From 1efac391ecd730798083d6aa618fc79055ce0b97 Mon Sep 17 00:00:00 2001 From: Rupal Mahajan Date: Wed, 30 Nov 2022 00:48:48 +0000 Subject: [PATCH 1/4] Fetch allowLeadingWildcards from config Signed-off-by: Rupal Mahajan --- .../public/components/utils/settings_service.ts | 2 ++ dashboards-reports/server/routes/lib/createReport.ts | 4 ++++ dashboards-reports/server/routes/report.ts | 4 +++- .../server/routes/utils/dataReportHelpers.ts | 12 +++++++++--- .../server/routes/utils/savedSearchReportHelper.ts | 8 +++++--- 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/dashboards-reports/public/components/utils/settings_service.ts b/dashboards-reports/public/components/utils/settings_service.ts index 197e5e4e..e752a7f6 100644 --- a/dashboards-reports/public/components/utils/settings_service.ts +++ b/dashboards-reports/public/components/utils/settings_service.ts @@ -22,10 +22,12 @@ export const uiSettingsService = { : rawTimeZone; const dateFormat = this.get('dateFormat'); const csvSeparator = this.get('csv:separator'); + const allowLeadingWildcards = this.get('query:allowLeadingWildcards'); return { timezone, dateFormat, csvSeparator, + allowLeadingWildcards, }; }, }; diff --git a/dashboards-reports/server/routes/lib/createReport.ts b/dashboards-reports/server/routes/lib/createReport.ts index 57f2c5cd..764ad71b 100644 --- a/dashboards-reports/server/routes/lib/createReport.ts +++ b/dashboards-reports/server/routes/lib/createReport.ts @@ -48,6 +48,9 @@ export const createReport = async ( request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat; // @ts-ignore const csvSeparator = request.query.csvSeparator || ','; + // @ts-ignore + const allowLeadingWildcards = request.query.allowLeadingWildcards; + const protocol = config.get('osd_server', 'protocol'); const hostname = config.get('osd_server', 'hostname'); const port = config.get('osd_server', 'port'); @@ -76,6 +79,7 @@ export const createReport = async ( opensearchClient, dateFormat, csvSeparator, + allowLeadingWildcards, isScheduledTask, logger ); diff --git a/dashboards-reports/server/routes/report.ts b/dashboards-reports/server/routes/report.ts index 4c443a57..e8ef4884 100644 --- a/dashboards-reports/server/routes/report.ts +++ b/dashboards-reports/server/routes/report.ts @@ -38,6 +38,7 @@ export default function (router: IRouter, config: ReportingConfig) { timezone: schema.maybe(schema.string()), dateFormat: schema.maybe(schema.string()), csvSeparator: schema.maybe(schema.string()), + allowLeadingWildcards: schema.maybe(schema.string()), }), }, }, @@ -66,7 +67,6 @@ export default function (router: IRouter, config: ReportingConfig) { try { const reportData = await createReport(request, context, report, config); - // if not deliver to user himself , no need to send actual file data to client const delivery = report.report_definition.delivery; addToMetric('report', 'create', 'count', report); @@ -98,6 +98,7 @@ export default function (router: IRouter, config: ReportingConfig) { timezone: schema.string(), dateFormat: schema.string(), csvSeparator: schema.string(), + allowLeadingWildcards: schema.string(), }), }, }, @@ -164,6 +165,7 @@ export default function (router: IRouter, config: ReportingConfig) { timezone: schema.string(), dateFormat: schema.string(), csvSeparator: schema.string(), + allowLeadingWildcards: schema.string(), }), }, }, diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts index bc108832..56a7a98a 100644 --- a/dashboards-reports/server/routes/utils/dataReportHelpers.ts +++ b/dashboards-reports/server/routes/utils/dataReportHelpers.ts @@ -12,6 +12,7 @@ import { buildOpenSearchQuery, Filter, Query, + OpenSearchQueryConfig, } from '../../../../../src/plugins/data/common'; export var metaData = { @@ -49,16 +50,21 @@ export const getSelectedFields = async (columns) => { // Build the OpenSearch query from the meta data // is_count is set to 1 if we building the count query but 0 if we building the fetch data query -export const buildRequestBody = (report: any, is_count: number) => { +export const buildRequestBody = (report: any, allowLeadingWildcards: boolean, is_count: number) => { let esbBoolQuery = esb.boolQuery(); const searchSourceJSON = report._source.searchSourceJSON; - const savedObjectQuery: Query = JSON.parse(searchSourceJSON).query; const savedObjectFilter: Filter = JSON.parse(searchSourceJSON).filter; + const config: OpenSearchQueryConfig = { + allowLeadingWildcards: allowLeadingWildcards, + queryStringOptions: {}, + ignoreFilterIfFieldNotInIndex: false, + } const QueryFromSavedObject = buildOpenSearchQuery( undefined, savedObjectQuery, - savedObjectFilter + savedObjectFilter, + config ); // Add time range if (report._source.timeFieldName && report._source.timeFieldName.length > 0) { diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts index ce05b412..c17246b8 100644 --- a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts @@ -30,6 +30,7 @@ export async function createSavedSearchReport( client: ILegacyClusterClient | ILegacyScopedClusterClient, dateFormat: string, csvSeparator: string, + allowLeadingWildcards: boolean, isScheduledTask: boolean = true, logger: Logger ): Promise { @@ -43,6 +44,7 @@ export async function createSavedSearchReport( params.core_params, dateFormat, csvSeparator, + allowLeadingWildcards, isScheduledTask, logger ); @@ -130,6 +132,7 @@ async function generateReportData( params: any, dateFormat: string, csvSeparator: string, + allowLeadingWildcards: boolean, isScheduledTask: boolean, logger: Logger ) { @@ -144,8 +147,7 @@ async function generateReportData( if (total === 0) { return ''; } - - const reqBody = buildRequestBody(report, 0); + const reqBody = buildRequestBody(report, allowLeadingWildcards, 0); logger.info( `[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}` ); @@ -182,7 +184,7 @@ async function generateReportData( // Build the OpenSearch Count query to count the size of result async function getOpenSearchDataSize() { - const countReq = buildRequestBody(report, 1); + const countReq = buildRequestBody(report, allowLeadingWildcards, 1); return await callCluster( client, 'count', From 8f2118ff3bf5c6e23f1cd2dd99b51b53852fba05 Mon Sep 17 00:00:00 2001 From: Rupal Mahajan Date: Wed, 30 Nov 2022 01:44:55 +0000 Subject: [PATCH 2/4] Fix tests Signed-off-by: Rupal Mahajan --- dashboards-reports/server/routes/report.ts | 1 + .../__tests__/savedSearchReportHelper.test.ts | 17 +++++++++++++++++ .../server/routes/utils/dataReportHelpers.ts | 4 ++-- .../routes/utils/savedSearchReportHelper.ts | 1 + 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dashboards-reports/server/routes/report.ts b/dashboards-reports/server/routes/report.ts index e8ef4884..b0a3a61a 100644 --- a/dashboards-reports/server/routes/report.ts +++ b/dashboards-reports/server/routes/report.ts @@ -67,6 +67,7 @@ export default function (router: IRouter, config: ReportingConfig) { try { const reportData = await createReport(request, context, report, config); + // if not deliver to user himself , no need to send actual file data to client const delivery = report.report_definition.delivery; addToMetric('report', 'create', 'count', report); diff --git a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 7009dda3..b1dd44c1 100644 --- a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -64,6 +64,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -76,6 +77,7 @@ describe('test create saved search report', () => { mockOpenSearchClient([]), mockDateFormat, ',', + true, undefined, mockLogger ); @@ -87,6 +89,7 @@ describe('test create saved search report', () => { mockOpenSearchClient([]), mockDateFormat, ',', + true, undefined, mockLogger ); @@ -101,6 +104,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -121,6 +125,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -155,6 +160,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -192,6 +198,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -221,6 +228,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -254,6 +262,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -281,6 +290,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -305,6 +315,7 @@ describe('test create saved search report', () => { client, mockDateFormat, '|', + true, undefined, mockLogger ); @@ -338,6 +349,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -363,6 +375,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -394,6 +407,7 @@ describe('test create saved search report', () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -421,6 +435,7 @@ test('create report for data set contains null field value', async () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -452,6 +467,7 @@ test('create report for data set with metadata fields', async () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); @@ -506,6 +522,7 @@ test('create report with empty/one/multiple(list) date values', async () => { client, mockDateFormat, ',', + true, undefined, mockLogger ); diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts index 56a7a98a..a7bee2b9 100644 --- a/dashboards-reports/server/routes/utils/dataReportHelpers.ts +++ b/dashboards-reports/server/routes/utils/dataReportHelpers.ts @@ -55,7 +55,7 @@ export const buildRequestBody = (report: any, allowLeadingWildcards: boolean, is const searchSourceJSON = report._source.searchSourceJSON; const savedObjectQuery: Query = JSON.parse(searchSourceJSON).query; const savedObjectFilter: Filter = JSON.parse(searchSourceJSON).filter; - const config: OpenSearchQueryConfig = { + const savedObjectConfig: OpenSearchQueryConfig = { allowLeadingWildcards: allowLeadingWildcards, queryStringOptions: {}, ignoreFilterIfFieldNotInIndex: false, @@ -64,7 +64,7 @@ export const buildRequestBody = (report: any, allowLeadingWildcards: boolean, is undefined, savedObjectQuery, savedObjectFilter, - config + savedObjectConfig, ); // Add time range if (report._source.timeFieldName && report._source.timeFieldName.length > 0) { diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts index c17246b8..67d68468 100644 --- a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts @@ -147,6 +147,7 @@ async function generateReportData( if (total === 0) { return ''; } + const reqBody = buildRequestBody(report, allowLeadingWildcards, 0); logger.info( `[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}` From 50ef47c348c1994f57a6ee371e3737dafd7cd924 Mon Sep 17 00:00:00 2001 From: Rupal Mahajan Date: Wed, 30 Nov 2022 02:32:01 +0000 Subject: [PATCH 3/4] nit Signed-off-by: Rupal Mahajan --- dashboards-reports/server/routes/report.ts | 2 +- .../server/routes/utils/savedSearchReportHelper.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards-reports/server/routes/report.ts b/dashboards-reports/server/routes/report.ts index b0a3a61a..b071b10a 100644 --- a/dashboards-reports/server/routes/report.ts +++ b/dashboards-reports/server/routes/report.ts @@ -67,7 +67,7 @@ export default function (router: IRouter, config: ReportingConfig) { try { const reportData = await createReport(request, context, report, config); - + // if not deliver to user himself , no need to send actual file data to client const delivery = report.report_definition.delivery; addToMetric('report', 'create', 'count', report); diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts index 67d68468..b5c58bea 100644 --- a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts @@ -147,7 +147,7 @@ async function generateReportData( if (total === 0) { return ''; } - + const reqBody = buildRequestBody(report, allowLeadingWildcards, 0); logger.info( `[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}` From 4b88209d5137befde054071a1f1b2eb730561fd5 Mon Sep 17 00:00:00 2001 From: Rupal Mahajan Date: Wed, 30 Nov 2022 09:59:56 -0800 Subject: [PATCH 4/4] add default value for allowLeadingWildCards Co-authored-by: Joshua Li Signed-off-by: Rupal Mahajan --- dashboards-reports/server/routes/lib/createReport.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards-reports/server/routes/lib/createReport.ts b/dashboards-reports/server/routes/lib/createReport.ts index 764ad71b..857775c5 100644 --- a/dashboards-reports/server/routes/lib/createReport.ts +++ b/dashboards-reports/server/routes/lib/createReport.ts @@ -49,7 +49,7 @@ export const createReport = async ( // @ts-ignore const csvSeparator = request.query.csvSeparator || ','; // @ts-ignore - const allowLeadingWildcards = request.query.allowLeadingWildcards; + const allowLeadingWildcards = !!request.query.allowLeadingWildcards; const protocol = config.get('osd_server', 'protocol'); const hostname = config.get('osd_server', 'hostname');