Use advanced settings for leading wildcards in query for csv reports (#…

…549) * Fetch allowLeadingWildcards from config Signed-off-by: Rupal Mahajan <[email protected]> * Fix tests Signed-off-by: Rupal Mahajan <[email protected]> * nit Signed-off-by: Rupal Mahajan <[email protected]> * add default value for allowLeadingWildCards Co-authored-by: Joshua Li <[email protected]> Signed-off-by: Rupal Mahajan <[email protected]> Signed-off-by: Rupal Mahajan <[email protected]> Co-authored-by: Joshua Li <[email protected]>
opensearch-project · Nov 30, 2022 · 5b9e73b · 5b9e73b
1 parent 926dc7f
commit 5b9e73b
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 5 deletions.
diff --git a/dashboards-reports/public/components/utils/settings_service.ts b/dashboards-reports/public/components/utils/settings_service.ts
@@ -22,10 +22,12 @@ export const uiSettingsService = {
         : rawTimeZone;
     const dateFormat = this.get('dateFormat');
     const csvSeparator = this.get('csv:separator');
+    const allowLeadingWildcards = this.get('query:allowLeadingWildcards');
     return {
       timezone,
       dateFormat,
       csvSeparator,
+      allowLeadingWildcards,
     };
   },
 };
diff --git a/dashboards-reports/server/routes/lib/createReport.ts b/dashboards-reports/server/routes/lib/createReport.ts
@@ -48,6 +48,9 @@ export const createReport = async (
     request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat;
   // @ts-ignore
   const csvSeparator = request.query.csvSeparator || ',';
+  // @ts-ignore
+  const allowLeadingWildcards = !!request.query.allowLeadingWildcards;
+
   const protocol = config.get('osd_server', 'protocol');
   const hostname = config.get('osd_server', 'hostname');
   const port = config.get('osd_server', 'port');
@@ -76,6 +79,7 @@ export const createReport = async (
         opensearchClient,
         dateFormat,
         csvSeparator,
+        allowLeadingWildcards,
         isScheduledTask,
         logger
       );

diff --git a/dashboards-reports/server/routes/report.ts b/dashboards-reports/server/routes/report.ts
@@ -38,6 +38,7 @@ export default function (router: IRouter, config: ReportingConfig) {
           timezone: schema.maybe(schema.string()),
           dateFormat: schema.maybe(schema.string()),
           csvSeparator: schema.maybe(schema.string()),
+          allowLeadingWildcards: schema.maybe(schema.string()),
         }),
       },
     },
@@ -98,6 +99,7 @@ export default function (router: IRouter, config: ReportingConfig) {
           timezone: schema.string(),
           dateFormat: schema.string(),
           csvSeparator: schema.string(),
+          allowLeadingWildcards: schema.string(),
         }),
       },
     },
@@ -164,6 +166,7 @@ export default function (router: IRouter, config: ReportingConfig) {
           timezone: schema.string(),
           dateFormat: schema.string(),
           csvSeparator: schema.string(),
+          allowLeadingWildcards: schema.string(),
         }),
       },
     },

diff --git a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts
@@ -64,6 +64,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -76,6 +77,7 @@ describe('test create saved search report', () => {
       mockOpenSearchClient([]),
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -87,6 +89,7 @@ describe('test create saved search report', () => {
       mockOpenSearchClient([]),
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -101,6 +104,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -121,6 +125,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -155,6 +160,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -192,6 +198,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -221,6 +228,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -254,6 +262,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -281,6 +290,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -305,6 +315,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       '|',
+      true,
       undefined,
       mockLogger
     );
@@ -338,6 +349,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -363,6 +375,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -394,6 +407,7 @@ describe('test create saved search report', () => {
       client,
       mockDateFormat,
       ',',
+      true,
       undefined,
       mockLogger
     );
@@ -421,6 +435,7 @@ test('create report for data set contains null field value', async () => {
     client,
     mockDateFormat,
     ',',
+    true,
     undefined,
     mockLogger
   );
@@ -452,6 +467,7 @@ test('create report for data set with metadata fields', async () => {
     client,
     mockDateFormat,
     ',',
+    true,
     undefined,
     mockLogger
   );
@@ -506,6 +522,7 @@ test('create report with empty/one/multiple(list) date values', async () => {
     client,
     mockDateFormat,
     ',',
+    true,
     undefined,
     mockLogger
   );

diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts
@@ -12,6 +12,7 @@ import {
   buildOpenSearchQuery,
   Filter,
   Query,
+  OpenSearchQueryConfig,
 } from '../../../../../src/plugins/data/common';
 
 export var metaData = {
@@ -49,16 +50,21 @@ export const getSelectedFields = async (columns) => {
 
 // Build the OpenSearch query from the meta data
 // is_count is set to 1 if we building the count query but 0 if we building the fetch data query
-export const buildRequestBody = (report: any, is_count: number) => {
+export const buildRequestBody = (report: any, allowLeadingWildcards: boolean, is_count: number) => {
   let esbBoolQuery = esb.boolQuery();
   const searchSourceJSON = report._source.searchSourceJSON;
-
   const savedObjectQuery: Query = JSON.parse(searchSourceJSON).query;
   const savedObjectFilter: Filter = JSON.parse(searchSourceJSON).filter;
+  const savedObjectConfig: OpenSearchQueryConfig = {
+    allowLeadingWildcards: allowLeadingWildcards,
+    queryStringOptions: {},
+    ignoreFilterIfFieldNotInIndex: false,
+  }
   const QueryFromSavedObject = buildOpenSearchQuery(
     undefined,
     savedObjectQuery,
-    savedObjectFilter
+    savedObjectFilter,
+    savedObjectConfig,
   );
   // Add time range
   if (report._source.timeFieldName && report._source.timeFieldName.length > 0) {

diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts
@@ -30,6 +30,7 @@ export async function createSavedSearchReport(
   client: ILegacyClusterClient | ILegacyScopedClusterClient,
   dateFormat: string,
   csvSeparator: string,
+  allowLeadingWildcards: boolean,
   isScheduledTask: boolean = true,
   logger: Logger
 ): Promise<CreateReportResultType> {
@@ -43,6 +44,7 @@ export async function createSavedSearchReport(
     params.core_params,
     dateFormat,
     csvSeparator,
+    allowLeadingWildcards,
     isScheduledTask,
     logger
   );
@@ -130,6 +132,7 @@ async function generateReportData(
   params: any,
   dateFormat: string,
   csvSeparator: string,
+  allowLeadingWildcards: boolean,
   isScheduledTask: boolean,
   logger: Logger
 ) {
@@ -145,7 +148,7 @@ async function generateReportData(
     return '';
   }
 
-  const reqBody = buildRequestBody(report, 0);
+  const reqBody = buildRequestBody(report, allowLeadingWildcards, 0);
   logger.info(
     `[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}`
   );
@@ -182,7 +185,7 @@ async function generateReportData(
 
   // Build the OpenSearch Count query to count the size of result
   async function getOpenSearchDataSize() {
-    const countReq = buildRequestBody(report, 1);
+    const countReq = buildRequestBody(report, allowLeadingWildcards, 1);
     return await callCluster(
       client,
       'count',