fix: improved elastic search query filter #837 (#861)

* fix: improved elastic search query filter

* fix: updated mathJS package version && improved number check

* fix: fixed inconsistency between data list and count number

* fix: minor naming change

* fix: improved mapping for digit with floating points

* fix: revert back to mongo query if es db doesnt return any value

* fix: fixed pagination conflict between ES and mongo query

* fix: remove empty space in string value of condition filter with trim()

* fix: minor change for object name and removed redundant type

src/common/utils.ts

@@ -619,23 +619,23 @@ export const createFullfacetPipeline = <T, Y extends object>(
   fields: Y,
   facets: string[],
   subField = "",
-  esPids?: string[],
+  esEnabled = false,
 ): PipelineStage[] => {
-<<<<<<< HEAD
   const pipeline: PipelineStage[] = [];
   const facetMatch: Record<string, unknown> = {};
-=======
-  const pipeline = [];
-  const facetMatch: Record<string, unknown> = esPids
-    ? { _id: { $in: esPids } }
-    : {};
->>>>>>> b35ceca7 (fix: fix lint issue)
 
   Object.keys(fields).forEach((key) => {
     if (facets.includes(key)) {
       facetMatch[key] = searchExpression<T>(model, key, fields[key as keyof Y]);
     }
 
+    if (esEnabled) {
+      if (key === "mode") {
+        pipelineHandler.handleModeSearch(pipeline, fields, key, idField);
+      }
+      return;
+    }
+
     switch (key) {
       case "text":
         pipelineHandler.handleTextSearch(pipeline, model, fields, key);
@@ -929,11 +929,6 @@ const replaceLikeOperatorRecursive = (
   return output;
 };
 
-export const isObjectWithOneKey = (obj: object): boolean => {
-  const keys = Object.keys(obj);
-  return keys.length === 1;
-};
-
 export const sleep = (ms: number) => {
   return new Promise((resolve) => setTimeout(resolve, ms));
 };

src/datasets/datasets.service.ts

@@ -18,7 +18,6 @@ import {
   createFullfacetPipeline,
   createFullqueryFilter,
   extractMetadataKeys,
-  isObjectWithOneKey,
   parseLimitFilters,
 } from "src/common/utils";
 import { ElasticSearchService } from "src/elastic-search/elastic-search.service";
@@ -109,7 +108,12 @@ export class DatasetsService {
     };
     const modifiers: QueryOptions = parseLimitFilters(filter.limits);
 
-    if (!this.ESClient || !whereClause) {
+    const isFieldsEmpty = Object.keys(whereClause).length === 0;
+
+    // NOTE: if Elastic search DB is empty we should use default mongo query
+    const canPerformElasticSearchQueries = await this.isElasticSearchDBEmpty();
+
+    if (!this.ESClient || isFieldsEmpty || !canPerformElasticSearchQueries) {
       datasets = await this.datasetModel
         .find(whereClause, null, modifiers)
         .exec();
@@ -119,9 +123,8 @@ export class DatasetsService {
         modifiers.limit,
         modifiers.skip,
       );
-
       datasets = await this.datasetModel
-        .find({ _id: { $in: esResult.data } }, null, modifiers)
+        .find({ _id: { $in: esResult.data } })
         .exec();
     }
 
@@ -131,23 +134,19 @@ export class DatasetsService {
   async fullFacet(
     filters: IFacets<IDatasetFields>,
   ): Promise<Record<string, unknown>[]> {
+    let data;
+
     const fields = filters.fields ?? {};
     const facets = filters.facets ?? [];
 
     // NOTE: if fields contains no value, we should use mongo query to optimize performance.
-    // however, fields always contain mode key, so we need to check if there's more than one key
-    const isFieldsEmpty = isObjectWithOneKey(fields);
+    // however, fields always contain "mode" key, so we need to check if there's more than one key
+    const isFieldsEmpty = Object.keys(fields).length === 1;
 
-    let data;
-    if (this.ESClient && !isFieldsEmpty) {
-      const totalDocCount = await this.datasetModel.countDocuments();
-
-      const { data: esPids } = await this.ESClient.search(
-        fields as IDatasetFields,
-        totalDocCount,
-      );
+    // NOTE: if Elastic search DB is empty we should use default mongo query
+    const canPerformElasticSearchQueries = await this.isElasticSearchDBEmpty();
 
-      fields.mode = { _id: { $in: esPids } };
+    if (!this.ESClient || isFieldsEmpty || !canPerformElasticSearchQueries) {
       const pipeline = createFullfacetPipeline<DatasetDocument, IDatasetFields>(
         this.datasetModel,
         "pid",
@@ -158,15 +157,25 @@ export class DatasetsService {
 
       data = await this.datasetModel.aggregate(pipeline).exec();
     } else {
+      const { count: initialCount } = await this.ESClient.getCount();
+      const { totalCount, data: esPids } = await this.ESClient.search(
+        fields as IDatasetFields,
+        initialCount,
+      );
+
+      fields.mode = { _id: { $in: esPids } };
       const pipeline = createFullfacetPipeline<DatasetDocument, IDatasetFields>(
         this.datasetModel,
         "pid",
         fields,
         facets,
         "",
+        !!this.ESClient,
       );
-
       data = await this.datasetModel.aggregate(pipeline).exec();
+
+      // NOTE: below code is to overwrite totalCount with ES result
+      data[0].all = [{ totalSets: totalCount }];
     }
 
     return data;
@@ -488,4 +497,10 @@ export class DatasetsService {
       }
     }
   }
+
+  async isElasticSearchDBEmpty() {
+    if (!this.ESClient) return;
+    const count = await this.ESClient.getCount();
+    return count.count > 0;
+  }
 }

src/elastic-search/configuration/indexSetting.ts

@@ -21,9 +21,7 @@ export const special_character_filter: AnalysisPatternReplaceCharFilter = {
 };
 
 //Dynamic templates
-export const dynamic_template:
-  | Record<string, MappingDynamicTemplate>[]
-  | never = [
+export const dynamic_template: Record<string, MappingDynamicTemplate>[] = [
   {
     string_as_keyword: {
       path_match: "scientificMetadata.*.*",
@@ -35,16 +33,16 @@ export const dynamic_template:
     },
   },
   {
-    long_as_long: {
+    long_as_double: {
       path_match: "scientificMetadata.*.*",
       match_mapping_type: "long",
       mapping: {
-        type: "long",
+        type: "double",
+        coerce: true,
         ignore_malformed: true,
       },
     },
   },
-
   {
     date_as_keyword: {
       path_match: "scientificMetadata.*.*",

src/elastic-search/elastic-search.service.ts

@@ -176,6 +176,18 @@ export class ElasticSearchService implements OnModuleInit {
     return bulkResponse;
   }
 
+  async getCount(index = this.defaultIndex) {
+    try {
+      return await this.esService.count({ index });
+    } catch (error) {
+      Logger.error("getCount failed-> ElasticSearchService", error);
+      throw new HttpException(
+        `getCount failed-> ElasticSearchService ${error}`,
+        HttpStatus.BAD_REQUEST,
+      );
+    }
+  }
+
   async updateIndex(index = this.defaultIndex) {
     try {
       await this.esService.indices.close({
@@ -247,7 +259,8 @@ export class ElasticSearchService implements OnModuleInit {
       const searchOptions = {
         track_scores: true,
         body: searchQuery,
-        size: limit + skip,
+        from: skip,
+        size: limit,
         sort: [{ _score: { order: "desc" } }],
         min_score: defaultMinScore,
         track_total_hits: true,

src/elastic-search/helpers/utils.ts

@@ -70,11 +70,17 @@ export const convertToElasticSearchQuery = (
   for (const field in scientificQuery) {
     const query = scientificQuery[field] as Record<string, unknown>;
     const operation = Object.keys(query)[0];
-    const value = query[operation];
+    const value =
+      typeof query[operation] === "string"
+        ? (query[operation] as string).trim()
+        : query[operation];
+
     const esOperation = operation.replace("$", "");
 
-    // Example: trasnformedKey = "scientificMetadata.someKey.value"
-    // firstPart = "scientificMetadata" , middlePart = "someKey"
+    // NOTE-EXAMPLE:
+    // trasnformedKey = "scientificMetadata.someKey.value"
+    // firstPart = "scientificMetadata",
+    // middlePart = "someKey"
     const { transformedKey, firstPart, middlePart } = transformMiddleKey(field);
 
     let filter = {};

src/elastic-search/interfaces/es-common.type.ts

@@ -22,6 +22,13 @@ export interface IShould {
   };
 }
 
+export interface IBoolShould {
+  bool: {
+    should: IShould[];
+    minimum_should_match?: number;
+  };
+}
+
 export interface IFilter {
   terms?: {
     [key: string]: string[];
@@ -35,12 +42,15 @@ export interface IFilter {
       lte?: string | number;
     };
   };
+  match?: {
+    [key: string]: string | number;
+  };
   nested?: {
     path: string;
     query: {
       bool: {
         must: (
-          | { match?: { [key: string]: string } }
+          | { term?: { [key: string]: string } }
           | { range?: { [key: string]: { [key: string]: string | number } } }
         )[];
       };

src/elastic-search/providers/query-builder.service.ts

@@ -1,7 +1,12 @@
 import { Injectable, Logger } from "@nestjs/common";
 import { QueryDslQueryContainer } from "@elastic/elasticsearch/lib/api/types";
 import { IDatasetFields } from "src/datasets/interfaces/dataset-filters.interface";
-import { IFilter, IShould, ObjectType } from "../interfaces/es-common.type";
+import {
+  IBoolShould,
+  IFilter,
+  IShould,
+  ObjectType,
+} from "../interfaces/es-common.type";
 import { FilterFields, QueryFields } from "./fields.enum";
 import { mapScientificQuery } from "src/common/utils";
 import { IScientificFilter } from "src/common/interfaces/common.interface";
@@ -57,7 +62,7 @@ export class SearchQueryService {
 
       shouldFilter.push(ownerGroup, accessGroups);
     }
-    return shouldFilter;
+    return { bool: { should: shouldFilter, minimum_should_match: 1 } };
   }
 
   private buildTextQuery(text: string): QueryDslQueryContainer[] {
@@ -161,19 +166,13 @@ export class SearchQueryService {
 
   private constructFinalQuery(
     filter: IFilter[],
-    should: IShould[],
+    should: IBoolShould,
     query: QueryDslQueryContainer[],
   ) {
     const finalQuery = {
       query: {
         bool: {
-          filter,
-          should: {
-            bool: {
-              should,
-              minimum_should_match: 1,
-            },
-          },
+          filter: [...filter, should],
           must: query,
         },
       },