diff --git a/.eslintrc.js b/.eslintrc.js index b52b9cb1..8a80bb83 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -23,7 +23,19 @@ module.exports = { 'react/no-array-index-key': 'off', "import/no-extraneous-dependencies": ["error", { "devDependencies": true - }] + }], + 'max-len': [ + 'error', + { + code: 150, + ignoreComments: true, + ignoreTrailingComments: true, + ignoreUrls: true, + ignoreStrings: true, + ignoreTemplateLiterals: true, + }, + ], + 'no-return-assign': 'warn' }, overrides: [{ "files": ["src/**/*.test.js"], diff --git a/devHelper/docker/es6.Dockerfile b/devHelper/docker/es6.Dockerfile new file mode 100644 index 00000000..95fda820 --- /dev/null +++ b/devHelper/docker/es6.Dockerfile @@ -0,0 +1,51 @@ +# From https://gist.github.com/rluvaton/3a8d5953e1ad8236e8953c2e7691e5de + +FROM ubuntu:bionic-20220531 + +# Must be root to install the packages +USER root + +# Install required deps +RUN apt update +RUN apt -y install gnupg wget apt-transport-https coreutils java-common + +# Import Elasticsearch GPG Key +RUN wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - + +# Add Elasticsearch 6.x APT repository +# setting CPU architecture to be amd64 explicity as in case this is being built from ARM (which it should) it would find the elasticsearch package (elasticsearch 6.x doesn't have ARM binary) +RUN echo "deb [arch=amd64] https://artifacts.elastic.co/packages/6.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-6.x.list + +# update after elastic-search repo added +RUN apt-get update + +# Install ARM Amazon JDK +RUN wget https://corretto.aws/downloads/latest/amazon-corretto-8-aarch64-linux-jdk.deb -O amazon-jdk.deb +RUN dpkg --skip-same-version -i amazon-jdk.deb +RUN rm amazon-jdk.deb + +# Install Elasticsearch 6.x +RUN apt-get -y install elasticsearch + +# the user was created when installed the elasticsearch +# Must not be root: +# org.elasticsearch.bootstrap.StartupException: java.lang.RuntimeException: can not run elasticsearch as root +USER elasticsearch + +WORKDIR /usr/share/elasticsearch + +# Append the custom conf + +RUN echo "# ---------------------------------- CUSTOM -----------------------------------" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# Added because of the following error (TL;DR: X-Pack features are not supported in ARM):" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# > org.elasticsearch.bootstrap.StartupException:" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# > ElasticsearchException[X-Pack is not supported and Machine Learning is not available for [linux-aarch64];" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# > you can use the other X-Pack features (unsupported) by setting xpack.ml.enabled: false in elasticsearch.yml]" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "xpack.ml.enabled: false" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# Added because we want to listen to requests coming from computers in the network" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "network.host: 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml + + +ENTRYPOINT [ "./bin/elasticsearch" ] diff --git a/devHelper/docker/esearch.yml b/devHelper/docker/esearch.yml index 4cd7d9ae..5d686729 100644 --- a/devHelper/docker/esearch.yml +++ b/devHelper/docker/esearch.yml @@ -3,7 +3,9 @@ version: "3.3" services: # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-prod-mode elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.12 + image: elasticsearch + build: + dockerfile: es6.Dockerfile ports: - "9200:9200" - "9300:9300" diff --git a/devHelper/scripts/commands.sh b/devHelper/scripts/commands.sh index d73932ef..88b3156d 100755 --- a/devHelper/scripts/commands.sh +++ b/devHelper/scripts/commands.sh @@ -47,7 +47,7 @@ curl -iv -X PUT "${ESHOST}/${indexName}" \ } }, "analyzer": { - "ngram_analyzer": { + "ngram_analyzer": { "type": "custom", "tokenizer": "ngram_tokenizer", "filter": [ @@ -91,6 +91,7 @@ curl -iv -X PUT "${ESHOST}/${indexName}" \ "file_type": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }, "file_format": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }, "auth_resource_path": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }, + "consortium_id": { "type": "integer" }, "file_count": { "type": "integer" }, "whatever_lab_result_value": { "type": "float" }, "some_nested_array_field": { @@ -159,4 +160,3 @@ curl -iv -X PUT "${ESHOST}/${configIndexName}" \ function es_indices() { curl -X GET "${ESHOST}/_cat/indices?v" } - diff --git a/package-lock.json b/package-lock.json index 6f8617f9..37dbf641 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gen3/guppy", - "version": "0.16.1", + "version": "0.17.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@gen3/guppy", - "version": "0.16.1", + "version": "0.17.0", "license": "ISC", "dependencies": { "@apollo/server": "^4.7.5", @@ -30171,7 +30171,7 @@ "object-assign": "^4.1.1", "signal-exit": "^3.0.0", "string-width": "^4.2.3", - "strip-ansi": "6.0.1", + "strip-ansi": "^6.0.1", "wide-align": "^1.1.2" } }, @@ -35918,7 +35918,7 @@ "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "6.0.1" + "strip-ansi": "^6.0.1" }, "dependencies": { "emoji-regex": { diff --git a/package.json b/package.json index 96638a6c..93916454 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gen3/guppy", - "version": "0.16.1", + "version": "0.17.0", "description": "Server that support GraphQL queries on data from elasticsearch", "main": "src/server/server.js", "directories": { diff --git a/src/components/ConnectedFilter/index.jsx b/src/components/ConnectedFilter/index.jsx index b384501c..39ce33a5 100644 --- a/src/components/ConnectedFilter/index.jsx +++ b/src/components/ConnectedFilter/index.jsx @@ -10,7 +10,6 @@ import { } from './utils'; import { ENUM_ACCESSIBILITY } from '../Utils/const'; import { - askGuppyAboutAllFieldsAndOptions, askGuppyAboutArrayTypes, askGuppyForAggregationData, getAllFieldsFromFilterConfigs, @@ -28,10 +27,13 @@ class ConnectedFilter extends React.Component { super(props); const filterConfigsFields = getAllFieldsFromFilterConfigs(props.filterConfig.tabs); - let allFields = props.accessibleFieldCheckList - ? _.union(filterConfigsFields, props.accessibleFieldCheckList) - : filterConfigsFields; - allFields = _.union(allFields, this.props.extraAggsFields); + const filterConfigsRegularAggFields = filterConfigsFields.fields || []; + const filterConfigsAsTextAggFields = filterConfigsFields.asTextAggFields || []; + const allRegularAggFields = props.accessibleFieldCheckList + ? _.union(filterConfigsRegularAggFields, props.accessibleFieldCheckList) + : filterConfigsRegularAggFields; + // props.extraAggsFields are chart fields, use asTextAgg for all of them + const allAsTextAggFields = _.union(filterConfigsAsTextAggFields, this.props.extraAggsFields); this.initialTabsOptions = {}; let initialFilter = this.props.adminAppliedPreFilters; @@ -47,7 +49,8 @@ class ConnectedFilter extends React.Component { } this.state = { - allFields, + allRegularAggFields, + allAsTextAggFields, initialAggsData: {}, receivedAggsData: {}, accessibility: ENUM_ACCESSIBILITY.ALL, @@ -68,12 +71,13 @@ class ConnectedFilter extends React.Component { if (this.props.onFilterChange) { this.props.onFilterChange(this.state.adminAppliedPreFilters, this.state.accessibility); } - askGuppyAboutAllFieldsAndOptions( + askGuppyForAggregationData( this.props.guppyConfig.path, this.props.guppyConfig.type, - this.state.allFields, - this.state.accessibility, + this.state.allRegularAggFields, + this.state.allAsTextAggFields, this.state.filter, + this.state.accessibility, ) .then((res) => { if (!res.data) { @@ -129,7 +133,8 @@ class ConnectedFilter extends React.Component { askGuppyForAggregationData( this.props.guppyConfig.path, this.props.guppyConfig.type, - this.state.allFields, + this.state.allRegularAggFields, + this.state.allAsTextAggFields, mergedFilterResults, this.state.accessibility, ) @@ -146,11 +151,13 @@ class ConnectedFilter extends React.Component { } getTabsWithSearchFields() { - const newTabs = this.props.filterConfig.tabs.map(({ title, fields, searchFields }) => { + const newTabs = this.props.filterConfig.tabs.map(({ + title, fields, searchFields, asTextAggFields = [], + }) => { if (searchFields) { - return { title, fields: searchFields.concat(fields) }; + return { title, fields: searchFields.concat(fields).concat(asTextAggFields) }; } - return { title, fields }; + return { title, fields: fields.concat(asTextAggFields) }; }); return newTabs; } @@ -177,7 +184,7 @@ class ConnectedFilter extends React.Component { // Get filter values const allFilterValues = this.props.filterConfig.tabs.reduce( - (accumulator, tab) => ([...accumulator, ...tab.fields]), + (accumulator, tab) => ([...accumulator, ...tab.fields, ...tab.asTextAggFields || []]), [], ); @@ -266,9 +273,10 @@ class ConnectedFilter extends React.Component { } if (!processedTabsOptions || Object.keys(processedTabsOptions).length === 0) return null; const { fieldMapping } = this.props; - const tabs = this.props.filterConfig.tabs.map(({ fields, searchFields }, index) => { + const tabs = this.props.filterConfig.tabs.map(({ fields, searchFields, asTextAggFields = [] }, index) => { + const aggFields = _.union(fields, asTextAggFields); const sections = getFilterSections( - fields, + aggFields, searchFields, fieldMapping, processedTabsOptions, @@ -335,6 +343,7 @@ ConnectedFilter.propTypes = { tabs: PropTypes.arrayOf(PropTypes.shape({ title: PropTypes.string, fields: PropTypes.arrayOf(PropTypes.string), + asTextAggFields: PropTypes.arrayOf(PropTypes.string), searchFields: PropTypes.arrayOf(PropTypes.string), })), }).isRequired, diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index 548e179e..2d7f3f00 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -1,13 +1,6 @@ import flat from 'flat'; import { queryGuppyForRawDataAndTotalCounts } from '../Utils/queries'; -export const getFilterGroupConfig = (filterConfig) => ({ - tabs: filterConfig.tabs.map((t) => ({ - title: t.title, - fields: t.filters.map((f) => f.field), - })), -}); - const getSingleFilterOption = (histogramResult, initHistogramRes, filterValuesToHide) => { if (!histogramResult || !histogramResult.histogram) { throw new Error(`Error parsing field options ${JSON.stringify(histogramResult)}`); @@ -28,14 +21,14 @@ const getSingleFilterOption = (histogramResult, initHistogramRes, filterValuesTo }); return rangeOptions; } - let rawtextOptions = histogramResult.histogram; + let rawTextOptions = histogramResult.histogram; // hide filterValuesToHide from filters // filterValuesToHide added to guppyConfig in data-portal if (filterValuesToHide.length > 0) { - rawtextOptions = histogramResult.histogram + rawTextOptions = histogramResult.histogram .filter((item) => filterValuesToHide.indexOf(item.key) < 0); } - const textOptions = rawtextOptions.map((item) => ({ + const textOptions = rawTextOptions.map((item) => ({ text: item.key, filterType: 'singleSelect', count: item.count, @@ -109,7 +102,7 @@ export const checkIsArrayField = (field, arrayFields) => { }; export const getFilterSections = ( - fields, + aggFields, searchFields, fieldMapping, tabsOptions, @@ -156,7 +149,7 @@ export const getFilterSections = ( }); } - const sections = fields.map((field) => { + const sections = aggFields.map((field) => { const overrideName = fieldMapping.find((entry) => (entry.field === field)); const label = overrideName ? overrideName.name : capitalizeFirstLetter(field); @@ -189,7 +182,7 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = const resultAggsData = {}; const flattenAggsData = flat(receivedAggsData, { safe: true }); Object.keys(flattenAggsData).forEach((field) => { - const actualFieldName = field.replace('.asTextHistogram', ''); + const actualFieldName = field.replace('.histogram', '').replace('.asTextHistogram', ''); const histogram = flattenAggsData[`${field}`]; if (!histogram) return; if (actualFieldName in filterResults) { diff --git a/src/components/GuppyWrapper/index.jsx b/src/components/GuppyWrapper/index.jsx index d58c0a9a..2303fd07 100644 --- a/src/components/GuppyWrapper/index.jsx +++ b/src/components/GuppyWrapper/index.jsx @@ -64,7 +64,8 @@ class GuppyWrapper extends React.Component { filter: { ...initialFilter }, rawData: [], totalCount: 0, - allFields: [], + allRegularAggFields: [], + allAsTextAggFields: [], rawDataFields: [], accessibleFieldObject: undefined, unaccessibleFieldObject: undefined, @@ -82,7 +83,7 @@ class GuppyWrapper extends React.Component { const rawDataFields = (this.props.rawDataFields && this.props.rawDataFields.length > 0) ? this.props.rawDataFields : fields; this.setState({ - allFields: fields, + allRegularAggFields: fields, rawDataFields, }, () => { this.getDataFromGuppy(this.state.rawDataFields, undefined, true); @@ -326,7 +327,8 @@ class GuppyWrapper extends React.Component { fetchAndUpdateRawData: this.handleFetchAndUpdateRawData.bind(this), downloadRawData: this.handleDownloadRawData.bind(this), downloadRawDataByFields: this.handleDownloadRawDataByFields.bind(this), - allFields: this.state.allFields, + allRegularAggFields: this.state.allRegularAggFields, + allAsTextAggFields: this.state.allAsTextAggFields, accessibleFieldObject: this.state.accessibleFieldObject, unaccessibleFieldObject: this.state.unaccessibleFieldObject, diff --git a/src/components/Utils/filters.js b/src/components/Utils/filters.js index 312a3d5d..b7c4c6d4 100644 --- a/src/components/Utils/filters.js +++ b/src/components/Utils/filters.js @@ -50,14 +50,14 @@ export const updateCountsInInitialTabsOptions = ( try { // flatten the tab options first // { - // project_id.asTextHistogram: ... - // visit.visit_label.asTextHistogram: ... + // project_id.histogram: ... + // visit.visit_label.histogram: ... // } const flattenInitialTabsOptions = flat(initialTabsOptions, { safe: true }); const flattenProcessedTabsOptions = flat(processedTabsOptions, { safe: true }); Object.keys(flattenInitialTabsOptions).forEach((field) => { - // in flattened tab options, to get actual field name, strip off the last '.asTextHistogram' - const actualFieldName = field.replace('.asTextHistogram', ''); + // in flattened tab options, to get actual field name, strip off the last '.histogram' or '.asTextHistogram' + const actualFieldName = field.replace('.histogram', '').replace('.asTextHistogram', ''); // check if Filter Value if not skip if (!allFilterValues.includes(actualFieldName)) { @@ -190,7 +190,7 @@ export const buildFilterStatusForURLFilter = (userFilter, tabs) => { const filterStatusArray = tabs.map(() => ([])); for (let tabIndex = 0; tabIndex < tabs.length; tabIndex += 1) { - const allFieldsForThisTab = tabs[tabIndex].fields; + const allFieldsForThisTab = _.union(tabs[tabIndex].fields, tabs[tabIndex].asTextAggFields || []); filterStatusArray[tabIndex] = allFieldsForThisTab.map(() => ({})); for (let i = 0; i < filteringFields.length; i += 1) { const sectionIndex = allFieldsForThisTab.indexOf(filteringFields[i]); diff --git a/src/components/Utils/queries.js b/src/components/Utils/queries.js index 91229706..48edb520 100644 --- a/src/components/Utils/queries.js +++ b/src/components/Utils/queries.js @@ -5,13 +5,13 @@ const graphqlEndpoint = '/graphql'; const downloadEndpoint = '/download'; const statusEndpoint = '/_status'; -const histogramQueryStrForEachField = (field) => { +const histogramQueryStrForEachField = (field, isAsTextAgg = false) => { const splittedFieldArray = field.split('.'); const splittedField = splittedFieldArray.shift(); if (splittedFieldArray.length === 0) { return (` ${splittedField} { - asTextHistogram { + ${(isAsTextAgg) ? 'asTextHistogram' : 'histogram'} { key count } @@ -23,7 +23,7 @@ const histogramQueryStrForEachField = (field) => { }`); }; -const queryGuppyForAggs = (path, type, fields, gqlFilter, acc) => { +const queryGuppyForAggs = (path, type, regularAggFields, asTextAggFields, gqlFilter, acc) => { let accessibility = acc; if (accessibility !== 'all' && accessibility !== 'accessible' && accessibility !== 'unaccessible') { accessibility = 'all'; @@ -34,7 +34,8 @@ const queryGuppyForAggs = (path, type, fields, gqlFilter, acc) => { const queryWithFilter = `query ($filter: JSON) { _aggregation { ${type} (filter: $filter, filterSelf: false, accessibility: ${accessibility}) { - ${fields.map((field) => histogramQueryStrForEachField(field))} + ${regularAggFields.map((field) => histogramQueryStrForEachField(field, false))} + ${asTextAggFields.map((field) => histogramQueryStrForEachField(field, true))} } } }`; @@ -44,7 +45,8 @@ const queryGuppyForAggs = (path, type, fields, gqlFilter, acc) => { queryBody.query = `query { _aggregation { ${type} (accessibility: ${accessibility}) { - ${fields.map((field) => histogramQueryStrForEachField(field))} + ${regularAggFields.map((field) => histogramQueryStrForEachField(field, false))} + ${asTextAggFields.map((field) => histogramQueryStrForEachField(field, true))} } } }`; @@ -244,7 +246,8 @@ export const getGQLFilter = (filterObj) => { } else if (filterValues.__combineMode && !hasSelectedValues && !hasRangeFilter) { // This filter only has a combine setting so far. We can ignore it. return; - } else { + } else if (hasSelectedValues) { + // filter has selected values but we don't know how to process it // eslint-disable-next-line no-console console.error(filterValues); throw new Error('Invalid filter object'); @@ -266,23 +269,19 @@ export const getGQLFilter = (filterObj) => { return gqlFilter; }; -export const askGuppyAboutAllFieldsAndOptions = (path, type, fields, accessibility, filter) => { - const gqlFilter = getGQLFilter(filter); - return queryGuppyForAggs(path, type, fields, gqlFilter, accessibility); -}; - // eslint-disable-next-line max-len export const askGuppyAboutArrayTypes = (path) => queryGuppyForStatus(path).then((res) => res.indices); export const askGuppyForAggregationData = ( path, type, - fields, + regularAggFields, + asTextAggFields, filter, accessibility, ) => { const gqlFilter = getGQLFilter(filter); - return queryGuppyForAggs(path, type, fields, gqlFilter, accessibility); + return queryGuppyForAggs(path, type, regularAggFields, asTextAggFields, gqlFilter, accessibility); }; export const askGuppyForSubAggregationData = ( @@ -333,8 +332,12 @@ export const askGuppyForRawData = ( ); }; -export const getAllFieldsFromFilterConfigs = (filterTabConfigs) => filterTabConfigs - .reduce((acc, cur) => acc.concat(cur.fields), []); +export const getAllFieldsFromFilterConfigs = (filterTabConfigs) => filterTabConfigs.reduce((acc, cur) => { + Object.keys(cur) + .filter((key) => key === 'fields' || key === 'asTextAggFields') + .forEach((key) => acc[key] = acc[key].concat(cur[key], [])); + return acc; +}, { fields: [], asTextAggFields: [] }); /** * Download all data from guppy using fields, filter, and sort args. diff --git a/src/components/__tests__/filters.test.js b/src/components/__tests__/filters.test.js index b414aef2..15751b9a 100644 --- a/src/components/__tests__/filters.test.js +++ b/src/components/__tests__/filters.test.js @@ -71,7 +71,7 @@ describe('can update a small set of tabs with new counts', () => { const initialTabsOptions = { annotated_sex: { - asTextHistogram: [ + histogram: [ { key: 'yellow', count: 137675 }, { key: 'pink', count: 56270 }, { key: 'silver', count: 2020 }, @@ -79,7 +79,7 @@ describe('can update a small set of tabs with new counts', () => { ], }, extra_data: { - asTextHistogram: [ + histogram: [ { key: 'a', count: 2 }, ], }, @@ -87,12 +87,12 @@ describe('can update a small set of tabs with new counts', () => { const processedTabsOptions = { annotated_sex: { - asTextHistogram: [ + histogram: [ { key: 'yellow', count: 1 }, { key: 'orange', count: 107574 }, ], }, - extra_data: { asTextHistogram: [] }, + extra_data: { histogram: [] }, }; const filtersApplied = { annotated_sex: { selectedValues: ['silver'] } }; @@ -132,7 +132,7 @@ describe('can update a small set of tabs with new counts, test with ranger slide ]; const initialTabsOptions = { field1: { - asTextHistogram: [ + histogram: [ { key: 'option1', count: 137675 }, { key: 'option2', count: 56270 }, { key: 'option3', count: 2020 }, @@ -140,7 +140,7 @@ describe('can update a small set of tabs with new counts, test with ranger slide ], }, field2: { - asTextHistogram: [ + histogram: [ { key: [0, 100], count: 100 }, ], }, @@ -148,12 +148,12 @@ describe('can update a small set of tabs with new counts, test with ranger slide const processedTabsOptions = { field1: { - asTextHistogram: [ + histogram: [ { key: 'option3', count: 30 }, ], }, field2: { - asTextHistogram: [ + histogram: [ { key: [4, 39], count: 49, diff --git a/stories/conf.js b/stories/conf.js index a8832450..adbdb419 100644 --- a/stories/conf.js +++ b/stories/conf.js @@ -5,6 +5,9 @@ export const filterConfig = { 'project', 'study', ], + asTextAggFields: [ + 'consortium_id', + ], }, { title: 'Subject',