diff --git a/package.json b/package.json index 043baaae..68f43504 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gen3/guppy", - "version": "0.1.2", + "version": "0.3.0", "description": "Server that support GraphQL queries on data from elasticsearch", "main": "src/server/server.js", "directories": { diff --git a/src/components/ConnectedFilter/index.jsx b/src/components/ConnectedFilter/index.jsx index 44db181a..2444d9aa 100644 --- a/src/components/ConnectedFilter/index.jsx +++ b/src/components/ConnectedFilter/index.jsx @@ -68,7 +68,9 @@ class ConnectedFilter extends React.Component { } setFilter(filter) { - this.filterGroupRef.current.resetFilter(); + if (this.filterGroupRef.current) { + this.filterGroupRef.current.resetFilter(); + } this.handleFilterChange(filter); } diff --git a/src/components/GuppyWrapper/index.jsx b/src/components/GuppyWrapper/index.jsx index a2ad0837..b5a8d100 100644 --- a/src/components/GuppyWrapper/index.jsx +++ b/src/components/GuppyWrapper/index.jsx @@ -6,6 +6,7 @@ import { askGuppyForTotalCounts, getAllFieldsFromGuppy, getAccessibleResources, + askGuppyForNestedAggregationData, } from '../Utils/queries'; import { ENUM_ACCESSIBILITY } from '../Utils/const'; @@ -98,6 +99,39 @@ class GuppyWrapper extends React.Component { if (!fields || fields.length === 0) { return Promise.resolve({ data: [], totalCount: 0 }); } + + // nested aggregation + if (this.props.guppyConfig.mainField) { + const numericAggregation = this.props.guppyConfig.mainFieldIsNumeric; + // use missedNestedFields instead of termsNestedFields for performance + return askGuppyForNestedAggregationData( + this.props.guppyConfig.path, + this.props.guppyConfig.type, + this.props.guppyConfig.mainField, + numericAggregation, + [], + this.props.guppyConfig.aggFields, + this.filter, + this.state.accessibility, + ).then((res) => { + if (!res || !res.data) { + throw new Error(`Error getting raw ${this.props.guppyConfig.type} data from Guppy server ${this.props.guppyConfig.path}.`); + } + const data = res.data._aggregation[this.props.guppyConfig.type]; + const field = numericAggregation ? 'asTextHistogram' : 'histogram'; + const parsedData = data[this.props.guppyConfig.mainField][field]; + if (updateDataWhenReceive) { + this.setState({ + rawData: parsedData, + }); + } + return { + data: res.data, + }; + }); + } + + // non-nested aggregation return askGuppyForRawData( this.props.guppyConfig.path, this.props.guppyConfig.type, @@ -274,6 +308,9 @@ GuppyWrapper.propTypes = { guppyConfig: PropTypes.shape({ path: PropTypes.string, type: PropTypes.string, + mainField: PropTypes.string, + mainFieldIsNumeric: PropTypes.bool, + aggFields: PropTypes.array, }).isRequired, children: PropTypes.oneOfType([ PropTypes.arrayOf(PropTypes.node), @@ -285,7 +322,7 @@ GuppyWrapper.propTypes = { fields: PropTypes.arrayOf(PropTypes.string), })), }).isRequired, - rawDataFields: PropTypes.arrayOf(PropTypes.string).isRequired, + rawDataFields: PropTypes.arrayOf(PropTypes.string), onReceiveNewAggsData: PropTypes.func, onFilterChange: PropTypes.func, accessibleFieldCheckList: PropTypes.arrayOf(PropTypes.string), @@ -294,6 +331,7 @@ GuppyWrapper.propTypes = { GuppyWrapper.defaultProps = { onReceiveNewAggsData: () => {}, onFilterChange: () => {}, + rawDataFields: [], accessibleFieldCheckList: undefined, }; diff --git a/src/components/Utils/queries.js b/src/components/Utils/queries.js index 3a35bd91..7e90fa7d 100644 --- a/src/components/Utils/queries.js +++ b/src/components/Utils/queries.js @@ -45,9 +45,9 @@ const queryGuppyForAggs = (path, type, fields, gqlFilter, acc) => { }).then(response => response.json()); }; -const nestedHistogramQueryStrForEachField = mainField => (` +const nestedHistogramQueryStrForEachField = (mainField, numericAggAsText) => (` ${mainField} { - histogram { + ${numericAggAsText ? 'asTextHistogram' : 'histogram'} { key count missingFields { @@ -68,6 +68,7 @@ const queryGuppyForNestedAgg = ( path, type, mainField, + numericAggAsText = false, termsFields, missingFields, gqlFilter, @@ -89,7 +90,7 @@ const queryGuppyForNestedAgg = ( const query = `query ($nestedAggFields: JSON) { _aggregation { ${type} (nestedAggFields: $nestedAggFields, accessibility: ${accessibility}) { - ${nestedHistogramQueryStrForEachField(mainField)} + ${nestedHistogramQueryStrForEachField(mainField, numericAggAsText)} } } }`; @@ -99,7 +100,7 @@ const queryGuppyForNestedAgg = ( const queryWithFilter = `query ($filter: JSON, $nestedAggFields: JSON) { _aggregation { ${type} (filter: $filter, filterSelf: false, nestedAggFields: $nestedAggFields, accessibility: ${accessibility}) { - ${nestedHistogramQueryStrForEachField(mainField)} + ${nestedHistogramQueryStrForEachField(mainField, numericAggAsText)} } } }`; @@ -112,7 +113,10 @@ const queryGuppyForNestedAgg = ( 'Content-Type': 'application/json', }, body: JSON.stringify(queryBody), - }).then(response => response.json()); + }).then(response => response.json()) + .catch((err) => { + throw new Error(`Error during queryGuppyForNestedAgg ${err}`); + }); }; const queryGuppyForRawDataAndTotalCounts = ( @@ -157,7 +161,10 @@ const queryGuppyForRawDataAndTotalCounts = ( 'Content-Type': 'application/json', }, body: JSON.stringify(queryBody), - }).then(response => response.json()); + }).then(response => response.json()) + .catch((err) => { + throw new Error(`Error during queryGuppyForRawDataAndTotalCounts ${err}`); + }); }; export const askGuppyAboutAllFieldsAndOptions = ( @@ -200,6 +207,7 @@ export const askGuppyForNestedAggregationData = ( path, type, mainField, + numericAggAsText, termsNestedFields, missedNestedFields, filter, @@ -210,6 +218,7 @@ export const askGuppyForNestedAggregationData = ( path, type, mainField, + numericAggAsText, termsNestedFields, missedNestedFields, gqlFilter, diff --git a/src/server/__mocks__/mockESData/mockNestedAggs.js b/src/server/__mocks__/mockESData/mockNestedAggs.js index 6ff5650a..c81622f2 100644 --- a/src/server/__mocks__/mockESData/mockNestedAggs.js +++ b/src/server/__mocks__/mockESData/mockNestedAggs.js @@ -12,7 +12,6 @@ const mockNestedAggs = () => { project: { terms: { field: 'project', - missing: 'no data', }, }, }, @@ -82,7 +81,6 @@ const mockNestedAggs = () => { project: { terms: { field: 'project', - missing: 'no data', }, }, }, @@ -181,7 +179,6 @@ const mockNestedAggs = () => { project: { terms: { field: 'project', - missing: 'no data', }, }, }, diff --git a/src/server/es/aggs.js b/src/server/es/aggs.js index 43815cdf..59816827 100644 --- a/src/server/es/aggs.js +++ b/src/server/es/aggs.js @@ -7,6 +7,78 @@ import { } from './const'; import config from '../config'; +const updateAggObjectForTermsFields = (termsFields, aggsObj) => { + const newAggsObj = { ...aggsObj }; + termsFields.forEach((element) => { + const variableName = `${element}Terms`; + newAggsObj[variableName] = { + terms: { + field: element, + }, + }; + }); + return newAggsObj; +}; + +const updateAggObjectForMissingFields = (missingFields, aggsObj) => { + const newAggsObj = { ...aggsObj }; + missingFields.forEach((element) => { + const variableName = `${element}Missing`; + newAggsObj[variableName] = { + missing: { + field: element, + }, + }; + }); + return newAggsObj; +}; + +const processResultsForNestedAgg = (nestedAggFields, item, resultObj) => { + let missingFieldResult; + if (nestedAggFields && nestedAggFields.missingFields) { + missingFieldResult = []; + nestedAggFields.missingFields.forEach((element) => { + const variableName = `${element}Missing`; + missingFieldResult.push({ + field: element, + count: item[variableName].doc_count, + }); + }); + } + + let termsFieldResult; + if (nestedAggFields && nestedAggFields.termsFields) { + termsFieldResult = []; + nestedAggFields.termsFields.forEach((element) => { + const tempResult = {}; + tempResult.field = element; + tempResult.terms = []; + const variableName = `${element}Terms`; + if (item[variableName].buckets && item[variableName].buckets.length > 0) { + item[variableName].buckets.forEach((itemElement) => { + tempResult.terms.push({ + key: itemElement.key, + count: itemElement.doc_count, + }); + }); + } else { + tempResult.terms.push({ + key: null, + count: 0, + }); + } + termsFieldResult.push(tempResult); + }); + } + + const newResultObj = { + ...resultObj, + ...(missingFieldResult && { missingFields: missingFieldResult }), + ...(termsFieldResult && { termsFields: termsFieldResult }), + }; + return newResultObj; +}; + /** * This function appends extra range limitation onto a query body "oldQuery" * export for test @@ -73,6 +145,7 @@ export const numericGlobalStats = async ( rangeEnd, filterSelf, defaultAuthFilter, + nestedAggFields, }) => { const queryBody = { size: 0 }; if (!!filter || !!defaultAuthFilter) { @@ -81,11 +154,17 @@ export const numericGlobalStats = async ( ); } queryBody.query = appendAdditionalRangeQuery(field, queryBody.query, rangeStart, rangeEnd); - const aggsObj = { + let aggsObj = { [AGGS_GLOBAL_STATS_NAME]: { stats: { field }, }, }; + if (nestedAggFields && nestedAggFields.termsFields) { + aggsObj = updateAggObjectForTermsFields(nestedAggFields.termsFields, aggsObj); + } + if (nestedAggFields && nestedAggFields.missingFields) { + aggsObj = updateAggObjectForMissingFields(nestedAggFields.missingFields, aggsObj); + } queryBody.aggs = aggsObj; const result = await esInstance.query(esIndex, esType, queryBody); let resultStats = result.aggregations[AGGS_GLOBAL_STATS_NAME]; @@ -97,6 +176,7 @@ export const numericGlobalStats = async ( key: range, ...resultStats, }; + resultStats = processResultsForNestedAgg(nestedAggFields, result.aggregations, resultStats); return resultStats; }; @@ -128,6 +208,7 @@ export const numericHistogramWithFixedRangeStep = async ( rangeStep, filterSelf, defaultAuthFilter, + nestedAggFields, }) => { const queryBody = { size: 0 }; if (!!filter || !!defaultAuthFilter) { @@ -139,6 +220,7 @@ export const numericHistogramWithFixedRangeStep = async ( field, filterSelf, defaultAuthFilter, + nestedAggFields, ); } queryBody.query = appendAdditionalRangeQuery(field, queryBody.query, rangeStart, rangeEnd); @@ -167,13 +249,31 @@ export const numericHistogramWithFixedRangeStep = async ( } aggsObj[AGGS_QUERY_NAME].histogram.offset = offset; } + if (nestedAggFields && nestedAggFields.termsFields) { + aggsObj[AGGS_QUERY_NAME].aggs = updateAggObjectForTermsFields( + nestedAggFields.termsFields, + aggsObj[AGGS_QUERY_NAME].aggs, + ); + } + if (nestedAggFields && nestedAggFields.missingFields) { + aggsObj[AGGS_QUERY_NAME].aggs = updateAggObjectForMissingFields( + nestedAggFields.missingFields, + aggsObj[AGGS_QUERY_NAME].aggs, + ); + } queryBody.aggs = aggsObj; const result = await esInstance.query(esIndex, esType, queryBody); - const parsedAggsResult = result.aggregations[AGGS_QUERY_NAME].buckets.map(item => ({ - key: [item.key, item.key + rangeStep], - ...item[AGGS_ITEM_STATS_NAME], - })); - return parsedAggsResult; + const finalResults = []; + let resultObj; + result.aggregations[AGGS_QUERY_NAME].buckets.forEach((item) => { + resultObj = processResultsForNestedAgg(nestedAggFields, item, resultObj); + finalResults.push({ + key: [item.key, item.key + rangeStep], + ...item[AGGS_ITEM_STATS_NAME], + ...resultObj, + }); + }); + return finalResults; }; /** @@ -204,6 +304,7 @@ export const numericHistogramWithFixedBinCount = async ( binCount, filterSelf, defaultAuthFilter, + nestedAggFields, }) => { const globalStats = await numericGlobalStats( { @@ -218,6 +319,7 @@ export const numericHistogramWithFixedBinCount = async ( rangeEnd, filterSelf, defaultAuthFilter, + nestedAggFields, }, ); const { min, max } = globalStats; @@ -238,6 +340,7 @@ export const numericHistogramWithFixedBinCount = async ( rangeStep, filterSelf, defaultAuthFilter, + nestedAggFields, }, ); }; @@ -271,6 +374,7 @@ export const numericAggregation = async ( binCount, filterSelf, defaultAuthFilter, + nestedAggFields, }, ) => { if (rangeStep <= 0) { @@ -302,6 +406,7 @@ export const numericAggregation = async ( rangeStep, filterSelf, defaultAuthFilter, + nestedAggFields, }, ); } @@ -320,6 +425,7 @@ export const numericAggregation = async ( binCount, filterSelf, defaultAuthFilter, + nestedAggFields, }, ); } @@ -336,6 +442,7 @@ export const numericAggregation = async ( rangeEnd, filterSelf, defaultAuthFilter, + nestedAggFields, }, ); return [result]; @@ -385,29 +492,15 @@ export const textAggregation = async ( missingAlias = { missing: config.esConfig.missingDataAlias }; } const aggsName = `${field}Aggs`; - const nestedAggQuery = {}; - if (nestedAggFields) { - nestedAggQuery.aggs = {}; - if (nestedAggFields.termsFields) { - nestedAggFields.termsFields.forEach((element) => { - const variableName = `${element}Terms`; - nestedAggQuery.aggs[variableName] = { - terms: { - field: element, - }, - }; - }); - } - if (nestedAggFields.missingFields) { - nestedAggFields.missingFields.forEach((element) => { - const variableName = `${element}Missing`; - nestedAggQuery.aggs[variableName] = { - missing: { - field: element, - }, - }; - }); - } + const aggsObj = {}; + if (nestedAggFields && nestedAggFields.termsFields) { + missingAlias = {}; + aggsObj.aggs = updateAggObjectForTermsFields(nestedAggFields.termsFields, aggsObj.aggs); + } + + if (nestedAggFields && nestedAggFields.missingFields) { + missingAlias = {}; + aggsObj.aggs = updateAggObjectForMissingFields(nestedAggFields.missingFields, aggsObj.aggs); } queryBody.aggs = { @@ -425,7 +518,7 @@ export const textAggregation = async ( ], size: PAGE_SIZE, }, - ...nestedAggQuery, + ...aggsObj, }, }; let resultSize; @@ -436,48 +529,11 @@ export const textAggregation = async ( resultSize = 0; result.aggregations[aggsName].buckets.forEach((item) => { - let missingFieldResult = undefined - if (nestedAggFields && nestedAggFields.missingFields) { - missingFieldResult = [] - nestedAggFields.missingFields.forEach((element) => { - const variableName = `${element}Missing`; - missingFieldResult.push({ - field: element, - count: item[variableName].doc_count, - }, - ); - }); - } - let termsFieldResult = undefined - if (nestedAggFields && nestedAggFields.termsFields) { - termsFieldResult = [] - nestedAggFields.termsFields.forEach((element) => { - let tempResult = {} - tempResult.field = element - tempResult.terms = [] - const variableName = `${element}Terms`; - if (item[variableName].buckets && item[variableName].buckets.length > 0) { - item[variableName].buckets.forEach((itemElement) => { - tempResult.terms.push({ - key: itemElement.key, - count: itemElement.doc_count, - }) - }) - } else { - tempResult.terms.push({ - key: null, - count: 0 - }) - } - termsFieldResult.push(tempResult) - }); - } - + const resultObj = processResultsForNestedAgg (nestedAggFields, item, resultObj) finalResults.push({ key: item.key[field], count: item.doc_count, - ...(missingFieldResult && {missingFields: missingFieldResult}), - ...(termsFieldResult && {termsFields: termsFieldResult}), + ...resultObj }); resultSize += 1; }); diff --git a/src/server/es/index.js b/src/server/es/index.js index 83ffc6a1..7dbfad49 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -373,6 +373,7 @@ class ES { binCount, filterSelf, defaultAuthFilter, + nestedAggFields, }) { return esAggregator.numericAggregation( { @@ -391,6 +392,7 @@ class ES { binCount, filterSelf, defaultAuthFilter, + nestedAggFields, }, ); } diff --git a/src/server/resolvers.js b/src/server/resolvers.js index 5f7aa04b..a60ecf32 100644 --- a/src/server/resolvers.js +++ b/src/server/resolvers.js @@ -80,14 +80,16 @@ const aggsTotalQueryResolver = (parent) => { const numericHistogramResolver = async (parent, args, context) => { const { esInstance, esIndex, esType, - filter, field, filterSelf, accessibility, + filter, field, nestedAggFields, filterSelf, accessibility, } = parent; + log.debug('[resolver.numericHistogramResolver] parent', parent); const { rangeStart, rangeEnd, rangeStep, binCount, } = args; const { authHelper } = context; const defaultAuthFilter = await authHelper.getDefaultFilter(accessibility); log.debug('[resolver.numericHistogramResolver] args', args); + const resultPromise = esInstance.numericAggregation({ esIndex, esType, @@ -99,6 +101,7 @@ const numericHistogramResolver = async (parent, args, context) => { binCount, filterSelf, defaultAuthFilter, + nestedAggFields, }); return resultPromise; }; @@ -217,6 +220,7 @@ const getResolver = (esConfig, esInstance) => { ...typeAggregationResolvers, HistogramForNumber: { histogram: numericHistogramResolver, + asTextHistogram: textHistogramResolver, }, HistogramForString: { histogram: textHistogramResolver, diff --git a/src/server/schema.js b/src/server/schema.js index b4bb7a6b..253da10d 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -183,12 +183,13 @@ export const buildSchemaString = (esConfig, esInstance) => { rangeEnd: Int, rangeStep: Int, binCount: Int, - ): [BucketsForNumber] + ): [BucketsForNestedNumberAgg], + asTextHistogram: [BucketsForNestedStringAgg] } `; const numberHistogramBucketSchema = ` - type BucketsForNumber { + type BucketsForNestedNumberAgg { """Lower and higher bounds for this bucket""" key: [Float] min: Float @@ -196,6 +197,8 @@ export const buildSchemaString = (esConfig, esInstance) => { avg: Float sum: Float count: Int + missingFields: [BucketsForNestedMissingFields] + termsFields: [BucketsForNestedTermsFields] } `;