Skip to content

Commit

Permalink
Performance optimization (#714)
Browse files Browse the repository at this point in the history 
* Keycloak user migration with role mappings

* document search performance and search optimization in general (#687)

* change in the replaceAll function usage (#691)
  • Loading branch information
@dinesh-aot
dinesh-aot committed Jun 2, 2023
1 parent fc65a02 commit 8ae0df2
Show file tree
Hide file tree
Showing 8 changed files with 160 additions and 103 deletions.
10 changes: 6 additions & 4 deletions api/aggregators/documentAggregator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ exports.createMatchAggr = async (schemaName, projectId, keywords, caseSensitive,
}

if (keywords) {
keywordModifier = { $text: { $search: keywords, $caseSensitive: caseSensitive } };
keywordModifier = { $text: { $search: "\""+keywords+"\"", $caseSensitive: caseSensitive} };
}

// query modifiers
Expand Down Expand Up @@ -142,7 +142,7 @@ exports.createMatchAggr = async (schemaName, projectId, keywords, caseSensitive,
* @param {array} roles Set of user roles
* @returns {array} Aggregate for documents.
*/
exports.createDocumentAggr = (populate, roles) => {
exports.createDocumentAggr = (populate, roles, sortingValue, sortField, sortDirection, pageNum, pageSize) => {
let aggregation = [];

// Allow documents to be sorted by status based on publish existence
Expand Down Expand Up @@ -185,9 +185,11 @@ exports.createDocumentAggr = (populate, roles) => {
}
});
}
var sortAggregation = aggregateHelper.createSortingPagingAggr('Document', sortingValue, sortField, sortDirection, pageNum, pageSize);
aggregation = [...aggregation, ...sortAggregation];

if (populate) {
// Handle project.
//Handle project.
aggregation.push(
{
'$lookup': {
Expand All @@ -199,7 +201,7 @@ exports.createDocumentAggr = (populate, roles) => {
},
{
'$addFields': {
project: '$project',
'project': '$project',
}
},
{
Expand Down
107 changes: 19 additions & 88 deletions api/aggregators/searchAggregator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ exports.createMatchAggr = async (schemaName, projectId, keywords, caseSensitive,
}

if (keywords) {
let keywordSearch = fuzzy && !keywords.startsWith("\"") && !keywords.endsWith("\"") ? fuzzySearch.createFuzzySearchString(keywords, 4, caseSensitive) : keywords;
keywords = keywords.replace(/"/g,"").trim();
let keywordSearch = fuzzy && !keywords.startsWith("\"") && !keywords.endsWith("\"") ? fuzzySearch.createFuzzySearchString(keywords, 4, caseSensitive) : "\""+ keywords +"\"";
keywordModifier = { $text: { $search: keywordSearch, $caseSensitive: caseSensitive } };
}

Expand Down Expand Up @@ -124,94 +125,24 @@ exports.createKeywordRegexAggr = function(decodedKeywords, schemaName) {
return keywordRegexFilter;
};

/**
* Create an aggregation that sets the sorting and paging for a query.
*
* @param {string} schemaName Schema being searched on
* @param {array} sortValues Values to sort by
* @param {string} sortField Single field to sort by
* @param {number} sortDirection Direction of sort
* @param {number} pageNum Page number to offset results by
* @param {number} pageSize Result set size
*
* @returns {array} Aggregation of sorting and paging
*/
exports.createSortingPagingAggr = function(schemaName, sortValues, sortField, sortDirection, pageNum, pageSize) {
const searchResultAggregation = [];
let datePostedHandlingTruncating = false;
if (sortField && sortValues !=null && typeof sortValues != "undefined" && sortField.includes(",") || Object.keys(sortValues).length > 1){
//sort will have multiple values passed
if (sortField.includes("datePosted") || Object.prototype.hasOwnProperty.call(sortValues, "datePosted")){
//datePosted is too specfic(in it's time) and needs the truncated form of date, can be expanded if other dates are required to be truncated
let tempSortValues = { };
for (let property in sortValues){
if (Object.prototype.hasOwnProperty.call(sortValues, property)) {
if (property === "datePosted"){
tempSortValues['date'] = sortValues[property];
} else {
tempSortValues[property] = sortValues[property];
}
}
}
sortValues = tempSortValues;
datePostedHandlingTruncating = true;
}

} else {
// if sortField is null, this would create a broken sort, so ignore it if its null
if(sortField && sortValues && sortValues[sortField]) {
sortValues[sortField] = sortDirection;
}
}

// if we have no sorting going on, we should sort by the score
if(!sortField) {
sortValues = { score: -1 };
}

// We don't want to have sort in the aggregation if the front end doesn't need sort.
if (sortField && sortDirection) {
if(datePostedHandlingTruncating){
// Currently this is just handling datePosted, if more date variables are needed change datePosted to a variable and detect it above
searchResultAggregation.push(

{ $addFields: {
'date':
{ $dateToString: {
'format': '%Y-%m-%d', 'date': '$datePosted'
}}

}},
{ $sort: sortValues }
);
} else {
searchResultAggregation.push(
{
$sort: sortValues
}
);
}
}

searchResultAggregation.push(
exports.createResultAggregator = function () {
return [
{
$skip: pageNum * pageSize
},
{
$limit: pageSize
},
);

const combinedAggregation = [{
$facet: {
searchResults: searchResultAggregation,
meta: [
{
$count: 'searchResultsTotal'
}
]
$facet: {
searchResults: [{
$match: {}
}],
meta: [
{ $limit: 1 },
{
$addFields: {
"searchResultsTotal": "$totalCount"
}
},
{ $project: { "searchResultsTotal": 1, "_id": 0 } }
]
}
}
}];

return combinedAggregation;
];
};
21 changes: 12 additions & 9 deletions api/controllers/search.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ const inspectionAggregator = require('../aggregators/inspectionAggregator');
const notificationProjectAggregator = require('../aggregators/notificationProjectAggregator');
const itemAggregator = require('../aggregators/itemAggregator');
const searchAggregator = require('../aggregators/searchAggregator');
const aggregateHelper = require('../helpers/aggregators');

const searchCollection = async function (roles, keywords, schemaName, pageNum, pageSize, project, projectLegislation, sortField = undefined, sortDirection = undefined, caseSensitive, populate = false, and, or, sortingValue, categorized, fuzzy) {
const aggregateCollation = {
Expand All @@ -34,7 +35,7 @@ const searchCollection = async function (roles, keywords, schemaName, pageNum, p
switch (schemaName) {
case constants.DOCUMENT:
matchAggregation = await documentAggregator.createMatchAggr(schemaName, project, decodedKeywords, caseSensitive, or, and, categorized, roles, fuzzy);
schemaAggregation = documentAggregator.createDocumentAggr(populate, roles,);
schemaAggregation = documentAggregator.createDocumentAggr(populate, roles, sortingValue, sortField, sortDirection, pageNum, pageSize);
break;
case constants.PROJECT:
matchAggregation = await searchAggregator.createMatchAggr(schemaName, project, decodedKeywords, caseSensitive, or, and, roles, fuzzy);
Expand Down Expand Up @@ -87,17 +88,19 @@ const searchCollection = async function (roles, keywords, schemaName, pageNum, p
}

// keyword regex
let keywordRegexFilter = !fuzzy && decodedKeywords ? searchAggregator.createKeywordRegexAggr(decodedKeywords, schemaName) : [];
let keywordRegexFilter = [];//!fuzzy && decodedKeywords ? searchAggregator.createKeywordRegexAggr(decodedKeywords, schemaName) : [];

// Create the sorting and paging aggregations.
const sortingPagingAggr = searchAggregator.createSortingPagingAggr(schemaName, sortingValue, sortField, sortDirection, pageNum, pageSize);
// For Document schema, the sorting and pagination pipelines have already been added for performance purpose
const resultAggr = (schemaName === constants.DOCUMENT?searchAggregator.createResultAggregator():
aggregateHelper.createSortingPagingAggr(schemaName, sortingValue, sortField, sortDirection, pageNum, pageSize));

// Combine all the aggregations.
let aggregation;
if (!schemaAggregation) {
aggregation = [...matchAggregation, ...keywordRegexFilter, ...sortingPagingAggr];
aggregation = [...matchAggregation, ...keywordRegexFilter, ...resultAggr];
} else {
aggregation = [...matchAggregation, ...schemaAggregation, ...keywordRegexFilter, ...sortingPagingAggr];
aggregation = [...matchAggregation, ...schemaAggregation, ...keywordRegexFilter, ...resultAggr];
}

return new Promise(function (resolve, reject) {
Expand Down Expand Up @@ -151,7 +154,7 @@ const executeQuery = async function (args, res) {
return Actions.sendResponse(res, 400, { });
}

Utils.recordAction('Search', keywords, args.swagger.params.auth_payload ? args.swagger.params.auth_payload.preferred_username : 'public');
await Utils.recordAction('Search', keywords, args.swagger.params.auth_payload ? args.swagger.params.auth_payload.preferred_username : 'public');

let sortDirection = undefined;
let sortField = undefined;
Expand Down Expand Up @@ -233,11 +236,11 @@ const executeQuery = async function (args, res) {

/***** Exported functions *****/
exports.publicGet = async function (args, res) {
executeQuery(args, res);
await executeQuery(args, res);
};

exports.protectedGet = function (args, res) {
executeQuery(args, res);
exports.protectedGet = async function (args, res) {
await executeQuery(args, res);
};

exports.protectedOptions = function (args, res) {
Expand Down
118 changes: 118 additions & 0 deletions api/helpers/aggregators.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,10 +448,128 @@ const isEmpty = (obj) => {
return true;
};

/**
* Create an aggregation that sets the sorting and paging for a query.
*
* @param {string} schemaName Name of the schema
* @param {array} sortValues Values to sort by
* @param {string} sortField Single field to sort by
* @param {number} sortDirection Direction of sort
* @param {number} pageNum Page number to offset results by
* @param {number} pageSize Result set size
*
* @returns {array} Aggregation of sorting and paging
*/
const createSortingPagingAggr = function(schemaName, sortValues, sortField, sortDirection, pageNum, pageSize) {
const searchResultAggregation = [];
let datePostedHandlingTruncating = false;
if (sortField && sortValues !=null && typeof sortValues != "undefined" && sortField.includes(",") || Object.keys(sortValues).length > 1){
//sort will have multiple values passed
if (sortField.includes("datePosted") || Object.prototype.hasOwnProperty.call(sortValues, "datePosted")){
//datePosted is too specfic(in it's time) and needs the truncated form of date, can be expanded if other dates are required to be truncated
let tempSortValues = { };
for (let property in sortValues){
if (Object.prototype.hasOwnProperty.call(sortValues, property)) {
if (property === "datePosted"){
tempSortValues['date'] = sortValues[property];
} else {
tempSortValues[property] = sortValues[property];
}
}
}
sortValues = tempSortValues;
datePostedHandlingTruncating = true;
}

} else {
// if sortField is null, this would create a broken sort, so ignore it if its null
if(sortField && sortValues && sortValues[sortField]) {
sortValues[sortField] = sortDirection;
}
}

// if we have no sorting going on, we should sort by the score
if(!sortField) {
sortValues = { score: -1 };
}

// We don't want to have sort in the aggregation if the front end doesn't need sort.
if (sortField && sortDirection) {
if(datePostedHandlingTruncating){
// Currently this is just handling datePosted, if more date variables are needed change datePosted to a variable and detect it above
searchResultAggregation.push(

{ $addFields: {
'date':
{ $dateToString: {
'format': '%Y-%m-%d', 'date': '$datePosted'
}}

}},
{ $sort: sortValues }
);
} else {
searchResultAggregation.push(
{
$sort: sortValues
}
);
}
}

searchResultAggregation.push(
{
$skip: pageNum * pageSize
},
{
$limit: pageSize
},
);

const combinedAggregation = [
{
$facet: {
searchResults: searchResultAggregation,
meta: [
{
$count: "searchResultsTotal"
}
]
}
}
];

// add a new field to store the totalCount which will later used to
// produce the searchResultsTotal in the final output
if(schemaName === constants.DOCUMENT) {
combinedAggregation.push({
$addFields: {
'searchResults.totalCount': {
$let: {
vars: {
item: {$arrayElemAt:["$meta",0]}
},
in: "$$item.searchResultsTotal"
}
}
}
},{
$unwind: {
path: '$searchResults'
}
},{
$replaceRoot: {newRoot: '$searchResults'}
});
}

return combinedAggregation;
};

// Exporting here so that the functions can be used in
// this file and exported.
exports.setProjectDefault = setProjectDefault;
exports.unwindProjectData = unwindProjectData;
exports.addProjectLookupAggrs = addProjectLookupAggrs;
exports.generateExpArray = generateExpArray;
exports.isEmpty = isEmpty;
exports.createSortingPagingAggr = createSortingPagingAggr;
1 change: 1 addition & 0 deletions app.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
'use strict';

require('dotenv').config();
var app = require('express')();
var fs = require('fs');
var uploadDir = process.env.UPLOAD_DIRECTORY || './uploads/';
Expand Down
2 changes: 1 addition & 1 deletion migrations/20190703105100-addProjectPhase.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ exports.setup = function(options, seedLink) {
seed = seedLink;
};

let listItems = require(process.cwd() + '/migrations_data/20190703105100-new-projectPhases.js');
let listItems = require(process.cwd() + '/migrations_data/lists/20190703105100-new-projectPhases.js');

exports.up = function(db) {
let mClient;
Expand Down
3 changes: 2 additions & 1 deletion openshift/kc_migration/custom_realm_users/script.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,8 @@ async function main() {
}
}

main();
main().catch((err)=>console.error('Migration end with error', JSON.stringify(err)))
.then(()=>console.log('Migration Completed Successfully'));

//returns user guid, please modify accordingly if required
//by default it uses idp specific user attribute to fetch the guid
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
"csv": "~5.1.1",
"db-migrate": "~0.11.4",
"db-migrate-mongodb": "~1.5.0",
"dotenv": "^16.0.1",
"epsg": "~0.5.0",
"express": "~4.16.0",
"flake-idgen": "~1.1.0",
Expand Down

0 comments on commit 8ae0df2

Please sign in to comment.