MIDRC-390 fix separate fields (#234)

* fix separate fields * fix things * fix filter * update version * more update * update storybook * fix filter * es6 docker image for apple silicon
uc-cdis · Sep 20, 2023 · f1431d1 · f1431d1
1 parent 7ee2eaa
commit f1431d1
Show file tree

Hide file tree

Showing 13 changed files with 144 additions and 69 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -23,7 +23,19 @@ module.exports = {
     'react/no-array-index-key': 'off',
     "import/no-extraneous-dependencies": ["error", {
       "devDependencies": true
-    }]
+    }],
+    'max-len': [
+      'error',
+      {
+        code: 150,
+        ignoreComments: true,
+        ignoreTrailingComments: true,
+        ignoreUrls: true,
+        ignoreStrings: true,
+        ignoreTemplateLiterals: true,
+      },
+    ],
+    'no-return-assign': 'warn'
   },
   overrides: [{
     "files": ["src/**/*.test.js"],

diff --git a/devHelper/docker/es6.Dockerfile b/devHelper/docker/es6.Dockerfile
@@ -0,0 +1,51 @@
+# From https://gist.github.com/rluvaton/3a8d5953e1ad8236e8953c2e7691e5de
+
+FROM ubuntu:bionic-20220531
+
+# Must be root to install the packages
+USER root
+
+# Install required deps
+RUN apt update
+RUN apt -y install gnupg wget apt-transport-https coreutils java-common
+
+# Import Elasticsearch GPG Key
+RUN wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
+
+# Add Elasticsearch 6.x APT repository
+# setting CPU architecture to be amd64 explicity as in case this is being built from ARM (which it should) it would find the elasticsearch package (elasticsearch 6.x doesn't have ARM binary)
+RUN echo "deb [arch=amd64] https://artifacts.elastic.co/packages/6.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-6.x.list
+
+# update after elastic-search repo added
+RUN apt-get update
+
+# Install ARM Amazon JDK
+RUN wget https://corretto.aws/downloads/latest/amazon-corretto-8-aarch64-linux-jdk.deb -O amazon-jdk.deb
+RUN dpkg --skip-same-version -i amazon-jdk.deb
+RUN rm amazon-jdk.deb
+
+# Install Elasticsearch 6.x
+RUN apt-get -y install elasticsearch
+
+# the user was created when installed the elasticsearch
+# Must not be root:
+# org.elasticsearch.bootstrap.StartupException: java.lang.RuntimeException: can not run elasticsearch as root
+USER elasticsearch
+
+WORKDIR /usr/share/elasticsearch
+
+# Append the custom conf
+
+RUN echo "# ---------------------------------- CUSTOM -----------------------------------"                                    >> /etc/elasticsearch/elasticsearch.yml
+RUN echo ""                                                                                                                   >> /etc/elasticsearch/elasticsearch.yml
+RUN echo "# Added because of the following error (TL;DR: X-Pack features are not supported in ARM):"                          >> /etc/elasticsearch/elasticsearch.yml
+RUN echo "# > org.elasticsearch.bootstrap.StartupException:"                                                                  >> /etc/elasticsearch/elasticsearch.yml
+RUN echo "# >   ElasticsearchException[X-Pack is not supported and Machine Learning is not available for [linux-aarch64];"    >> /etc/elasticsearch/elasticsearch.yml
+RUN echo "# >   you can use the other X-Pack features (unsupported) by setting xpack.ml.enabled: false in elasticsearch.yml]" >> /etc/elasticsearch/elasticsearch.yml
+RUN echo "xpack.ml.enabled: false"                                                                                            >> /etc/elasticsearch/elasticsearch.yml
+RUN echo ""                                                                                                                   >> /etc/elasticsearch/elasticsearch.yml
+RUN echo "# Added because we want to listen to requests coming from computers in the network"                                 >> /etc/elasticsearch/elasticsearch.yml
+RUN echo "network.host: 0.0.0.0"                                                                                              >> /etc/elasticsearch/elasticsearch.yml
+
+
+ENTRYPOINT [ "./bin/elasticsearch" ]
diff --git a/devHelper/docker/esearch.yml b/devHelper/docker/esearch.yml
@@ -3,7 +3,9 @@ version: "3.3"
 services:
   # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-prod-mode
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.12
+    image: elasticsearch
+    build:
+      dockerfile: es6.Dockerfile
     ports:
       - "9200:9200"
       - "9300:9300"

diff --git a/devHelper/scripts/commands.sh b/devHelper/scripts/commands.sh
@@ -47,7 +47,7 @@ curl -iv -X PUT "${ESHOST}/${indexName}" \
             }
           },
           "analyzer": {
-            "ngram_analyzer": { 
+            "ngram_analyzer": {
               "type": "custom",
               "tokenizer": "ngram_tokenizer",
               "filter": [
@@ -91,6 +91,7 @@ curl -iv -X PUT "${ESHOST}/${indexName}" \
           "file_type": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
           "file_format": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
           "auth_resource_path": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
+          "consortium_id": { "type": "integer" },
           "file_count": { "type": "integer" },
           "whatever_lab_result_value": { "type": "float" },
           "some_nested_array_field": {
@@ -159,4 +160,3 @@ curl -iv -X PUT "${ESHOST}/${configIndexName}" \
 function es_indices() {
   curl -X GET "${ESHOST}/_cat/indices?v"
 }
-
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@gen3/guppy",
-  "version": "0.16.1",
+  "version": "0.17.0",
   "description": "Server that support GraphQL queries on data from elasticsearch",
   "main": "src/server/server.js",
   "directories": {

diff --git a/src/components/ConnectedFilter/index.jsx b/src/components/ConnectedFilter/index.jsx
@@ -10,7 +10,6 @@ import {
 } from './utils';
 import { ENUM_ACCESSIBILITY } from '../Utils/const';
 import {
-  askGuppyAboutAllFieldsAndOptions,
   askGuppyAboutArrayTypes,
   askGuppyForAggregationData,
   getAllFieldsFromFilterConfigs,
@@ -28,10 +27,13 @@ class ConnectedFilter extends React.Component {
     super(props);
 
     const filterConfigsFields = getAllFieldsFromFilterConfigs(props.filterConfig.tabs);
-    let allFields = props.accessibleFieldCheckList
-      ? _.union(filterConfigsFields, props.accessibleFieldCheckList)
-      : filterConfigsFields;
-    allFields = _.union(allFields, this.props.extraAggsFields);
+    const filterConfigsRegularAggFields = filterConfigsFields.fields || [];
+    const filterConfigsAsTextAggFields = filterConfigsFields.asTextAggFields || [];
+    const allRegularAggFields = props.accessibleFieldCheckList
+      ? _.union(filterConfigsRegularAggFields, props.accessibleFieldCheckList)
+      : filterConfigsRegularAggFields;
+    // props.extraAggsFields are chart fields, use asTextAgg for all of them
+    const allAsTextAggFields = _.union(filterConfigsAsTextAggFields, this.props.extraAggsFields);
 
     this.initialTabsOptions = {};
     let initialFilter = this.props.adminAppliedPreFilters;
@@ -47,7 +49,8 @@ class ConnectedFilter extends React.Component {
     }
 
     this.state = {
-      allFields,
+      allRegularAggFields,
+      allAsTextAggFields,
       initialAggsData: {},
       receivedAggsData: {},
       accessibility: ENUM_ACCESSIBILITY.ALL,
@@ -68,12 +71,13 @@ class ConnectedFilter extends React.Component {
     if (this.props.onFilterChange) {
       this.props.onFilterChange(this.state.adminAppliedPreFilters, this.state.accessibility);
     }
-    askGuppyAboutAllFieldsAndOptions(
+    askGuppyForAggregationData(
       this.props.guppyConfig.path,
       this.props.guppyConfig.type,
-      this.state.allFields,
-      this.state.accessibility,
+      this.state.allRegularAggFields,
+      this.state.allAsTextAggFields,
       this.state.filter,
+      this.state.accessibility,
     )
       .then((res) => {
         if (!res.data) {
@@ -129,7 +133,8 @@ class ConnectedFilter extends React.Component {
     askGuppyForAggregationData(
       this.props.guppyConfig.path,
       this.props.guppyConfig.type,
-      this.state.allFields,
+      this.state.allRegularAggFields,
+      this.state.allAsTextAggFields,
       mergedFilterResults,
       this.state.accessibility,
     )
@@ -146,11 +151,13 @@ class ConnectedFilter extends React.Component {
   }
 
   getTabsWithSearchFields() {
-    const newTabs = this.props.filterConfig.tabs.map(({ title, fields, searchFields }) => {
+    const newTabs = this.props.filterConfig.tabs.map(({
+      title, fields, searchFields, asTextAggFields = [],
+    }) => {
       if (searchFields) {
-        return { title, fields: searchFields.concat(fields) };
+        return { title, fields: searchFields.concat(fields).concat(asTextAggFields) };
       }
-      return { title, fields };
+      return { title, fields: fields.concat(asTextAggFields) };
     });
     return newTabs;
   }
@@ -177,7 +184,7 @@ class ConnectedFilter extends React.Component {
 
     // Get filter values
     const allFilterValues = this.props.filterConfig.tabs.reduce(
-      (accumulator, tab) => ([...accumulator, ...tab.fields]),
+      (accumulator, tab) => ([...accumulator, ...tab.fields, ...tab.asTextAggFields || []]),
       [],
     );
 
@@ -266,9 +273,10 @@ class ConnectedFilter extends React.Component {
     }
     if (!processedTabsOptions || Object.keys(processedTabsOptions).length === 0) return null;
     const { fieldMapping } = this.props;
-    const tabs = this.props.filterConfig.tabs.map(({ fields, searchFields }, index) => {
+    const tabs = this.props.filterConfig.tabs.map(({ fields, searchFields, asTextAggFields = [] }, index) => {
+      const aggFields = _.union(fields, asTextAggFields);
       const sections = getFilterSections(
-        fields,
+        aggFields,
         searchFields,
         fieldMapping,
         processedTabsOptions,
@@ -335,6 +343,7 @@ ConnectedFilter.propTypes = {
     tabs: PropTypes.arrayOf(PropTypes.shape({
       title: PropTypes.string,
       fields: PropTypes.arrayOf(PropTypes.string),
+      asTextAggFields: PropTypes.arrayOf(PropTypes.string),
       searchFields: PropTypes.arrayOf(PropTypes.string),
     })),
   }).isRequired,

diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js
@@ -1,13 +1,6 @@
 import flat from 'flat';
 import { queryGuppyForRawDataAndTotalCounts } from '../Utils/queries';
 
-export const getFilterGroupConfig = (filterConfig) => ({
-  tabs: filterConfig.tabs.map((t) => ({
-    title: t.title,
-    fields: t.filters.map((f) => f.field),
-  })),
-});
-
 const getSingleFilterOption = (histogramResult, initHistogramRes, filterValuesToHide) => {
   if (!histogramResult || !histogramResult.histogram) {
     throw new Error(`Error parsing field options ${JSON.stringify(histogramResult)}`);
@@ -28,14 +21,14 @@ const getSingleFilterOption = (histogramResult, initHistogramRes, filterValuesTo
     });
     return rangeOptions;
   }
-  let rawtextOptions = histogramResult.histogram;
+  let rawTextOptions = histogramResult.histogram;
   // hide filterValuesToHide from filters
   // filterValuesToHide added to guppyConfig in data-portal
   if (filterValuesToHide.length > 0) {
-    rawtextOptions = histogramResult.histogram
+    rawTextOptions = histogramResult.histogram
       .filter((item) => filterValuesToHide.indexOf(item.key) < 0);
   }
-  const textOptions = rawtextOptions.map((item) => ({
+  const textOptions = rawTextOptions.map((item) => ({
     text: item.key,
     filterType: 'singleSelect',
     count: item.count,
@@ -109,7 +102,7 @@ export const checkIsArrayField = (field, arrayFields) => {
 };
 
 export const getFilterSections = (
-  fields,
+  aggFields,
   searchFields,
   fieldMapping,
   tabsOptions,
@@ -156,7 +149,7 @@ export const getFilterSections = (
     });
   }
 
-  const sections = fields.map((field) => {
+  const sections = aggFields.map((field) => {
     const overrideName = fieldMapping.find((entry) => (entry.field === field));
     const label = overrideName ? overrideName.name : capitalizeFirstLetter(field);
 
@@ -189,7 +182,7 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) =
   const resultAggsData = {};
   const flattenAggsData = flat(receivedAggsData, { safe: true });
   Object.keys(flattenAggsData).forEach((field) => {
-    const actualFieldName = field.replace('.asTextHistogram', '');
+    const actualFieldName = field.replace('.histogram', '').replace('.asTextHistogram', '');
     const histogram = flattenAggsData[`${field}`];
     if (!histogram) return;
     if (actualFieldName in filterResults) {

diff --git a/src/components/GuppyWrapper/index.jsx b/src/components/GuppyWrapper/index.jsx
@@ -64,7 +64,8 @@ class GuppyWrapper extends React.Component {
       filter: { ...initialFilter },
       rawData: [],
       totalCount: 0,
-      allFields: [],
+      allRegularAggFields: [],
+      allAsTextAggFields: [],
       rawDataFields: [],
       accessibleFieldObject: undefined,
       unaccessibleFieldObject: undefined,
@@ -82,7 +83,7 @@ class GuppyWrapper extends React.Component {
       const rawDataFields = (this.props.rawDataFields && this.props.rawDataFields.length > 0)
         ? this.props.rawDataFields : fields;
       this.setState({
-        allFields: fields,
+        allRegularAggFields: fields,
         rawDataFields,
       }, () => {
         this.getDataFromGuppy(this.state.rawDataFields, undefined, true);
@@ -326,7 +327,8 @@ class GuppyWrapper extends React.Component {
             fetchAndUpdateRawData: this.handleFetchAndUpdateRawData.bind(this),
             downloadRawData: this.handleDownloadRawData.bind(this),
             downloadRawDataByFields: this.handleDownloadRawDataByFields.bind(this),
-            allFields: this.state.allFields,
+            allRegularAggFields: this.state.allRegularAggFields,
+            allAsTextAggFields: this.state.allAsTextAggFields,
             accessibleFieldObject: this.state.accessibleFieldObject,
             unaccessibleFieldObject: this.state.unaccessibleFieldObject,
 

diff --git a/src/components/Utils/filters.js b/src/components/Utils/filters.js
@@ -50,14 +50,14 @@ export const updateCountsInInitialTabsOptions = (
   try {
     // flatten the tab options first
     // {
-    //   project_id.asTextHistogram: ...
-    //   visit.visit_label.asTextHistogram: ...
+    //   project_id.histogram: ...
+    //   visit.visit_label.histogram: ...
     // }
     const flattenInitialTabsOptions = flat(initialTabsOptions, { safe: true });
     const flattenProcessedTabsOptions = flat(processedTabsOptions, { safe: true });
     Object.keys(flattenInitialTabsOptions).forEach((field) => {
-      // in flattened tab options, to get actual field name, strip off the last '.asTextHistogram'
-      const actualFieldName = field.replace('.asTextHistogram', '');
+      // in flattened tab options, to get actual field name, strip off the last '.histogram' or '.asTextHistogram'
+      const actualFieldName = field.replace('.histogram', '').replace('.asTextHistogram', '');
 
       // check if Filter Value if not skip
       if (!allFilterValues.includes(actualFieldName)) {
@@ -190,7 +190,7 @@ export const buildFilterStatusForURLFilter = (userFilter, tabs) => {
   const filterStatusArray = tabs.map(() => ([]));
 
   for (let tabIndex = 0; tabIndex < tabs.length; tabIndex += 1) {
-    const allFieldsForThisTab = tabs[tabIndex].fields;
+    const allFieldsForThisTab = _.union(tabs[tabIndex].fields, tabs[tabIndex].asTextAggFields || []);
     filterStatusArray[tabIndex] = allFieldsForThisTab.map(() => ({}));
     for (let i = 0; i < filteringFields.length; i += 1) {
       const sectionIndex = allFieldsForThisTab.indexOf(filteringFields[i]);