From 22238271005c7e158c0d50f29e9f8e3d807e5c3c Mon Sep 17 00:00:00 2001 From: Lena Date: Tue, 28 Apr 2020 16:15:35 -0700 Subject: [PATCH 1/7] [Fix] Chart Sort/Min Val --- datahub/webapp/lib/chart/chart-meta-processing.ts | 6 ++++++ datahub/webapp/lib/chart/chart-utils.ts | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datahub/webapp/lib/chart/chart-meta-processing.ts b/datahub/webapp/lib/chart/chart-meta-processing.ts index e7c896701..229686048 100644 --- a/datahub/webapp/lib/chart/chart-meta-processing.ts +++ b/datahub/webapp/lib/chart/chart-meta-processing.ts @@ -299,6 +299,12 @@ function computeScaleOptions( ...axis.ticks, min: axisMeta.min, }; + } else { + // default min to 0 unless specified + axis.ticks = { + ...axis.ticks, + min: 0, + }; } } diff --git a/datahub/webapp/lib/chart/chart-utils.ts b/datahub/webapp/lib/chart/chart-utils.ts index 5480f387e..13481088c 100644 --- a/datahub/webapp/lib/chart/chart-utils.ts +++ b/datahub/webapp/lib/chart/chart-utils.ts @@ -62,5 +62,8 @@ export function sortTable( (a, b) => a[columnIndex] - b[columnIndex] ) as any[]; } - return tableRows.sort((a, b) => (a[columnIndex] > b[columnIndex] ? 1 : -1)); + + // TODO: do not sort categorical data until sort feature + // return tableRows.sort((a, b) => (a[columnIndex] > b[columnIndex] ? 1 : -1)); + return tableRows; } From 116ae5ab89d233828dbba587aa5251ed50831bb8 Mon Sep 17 00:00:00 2001 From: Lena Date: Wed, 29 Apr 2020 19:00:24 -0700 Subject: [PATCH 2/7] [Chart] Sort Feature --- datahub/config/datadoc.yaml | 3 + .../DataDocChartCell/DataDocChartCell.tsx | 6 +- .../DataDocChartCell/DataDocChartComposer.tsx | 67 ++++++++++++++++--- datahub/webapp/const/dataDocChart.ts | 6 ++ .../webapp/lib/chart/chart-data-processing.ts | 2 +- .../lib/chart/chart-data-transformation.ts | 23 +++++-- .../webapp/lib/chart/chart-meta-processing.ts | 4 ++ datahub/webapp/lib/chart/chart-utils.ts | 27 ++++++-- 8 files changed, 114 insertions(+), 24 deletions(-) diff --git a/datahub/config/datadoc.yaml b/datahub/config/datadoc.yaml index f3321100d..b3c5d6110 100644 --- a/datahub/config/datadoc.yaml +++ b/datahub/config/datadoc.yaml @@ -42,6 +42,9 @@ cell_types: scale: '' min: 0 max: 0 + sort: + idx: 0 + asc: true y_axis: label: '' scale: '' diff --git a/datahub/webapp/components/DataDocChartCell/DataDocChartCell.tsx b/datahub/webapp/components/DataDocChartCell/DataDocChartCell.tsx index 20bb8cf0e..653858e35 100644 --- a/datahub/webapp/components/DataDocChartCell/DataDocChartCell.tsx +++ b/datahub/webapp/components/DataDocChartCell/DataDocChartCell.tsx @@ -129,6 +129,8 @@ export const DataDocChartCell: React.FunctionComponent = ({ formatSeriesCol, formatValueCols, aggSeries, + sortIndex, + sortAsc, } = getDataTransformationOptions(meta); return transformData( @@ -138,7 +140,9 @@ export const DataDocChartCell: React.FunctionComponent = ({ formatAggCol, formatSeriesCol, formatValueCols, - aggSeries + aggSeries, + sortIndex, + sortAsc ); } } diff --git a/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx b/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx index 4b583f75a..2f6db774f 100644 --- a/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx +++ b/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx @@ -23,7 +23,7 @@ import { colorPalette, colorPaletteNames } from 'const/chartColors'; import { defaultReactSelectStyles } from 'lib/utils/react-select'; import { mapMetaToFormVals } from 'lib/chart/chart-meta-processing'; -import { transformData } from 'lib/chart/chart-data-transformation'; +import { transformData, sortData } from 'lib/chart/chart-data-transformation'; import { useChartSource } from 'hooks/chart/useChartSource'; import { DataDocChart } from './DataDocChart'; @@ -109,7 +109,9 @@ const DataDocChartComposerComponent: React.FunctionComponent< values.formatAggCol, values.formatSeriesCol, values.formatValueCols, - values.aggSeries + values.aggSeries, + values.sortIndex, + values.sortAsc ) : null; }, [ @@ -121,6 +123,33 @@ const DataDocChartComposerComponent: React.FunctionComponent< values.formatValueCols, values.aggType, values.aggSeries, + values.sortIndex, + values.sortAsc, + ]); + + const tableData: any[][] = React.useMemo(() => { + if (tableTab === 'original') { + if (values.sortIndex != null && statementResultData) { + return [ + statementResultData[0], + ...sortData( + statementResultData.slice(1), + values.sortIndex, + values.sortAsc + ), + ]; + } else { + return statementResultData; + } + } else { + return chartData; + } + }, [ + tableTab, + statementResultData, + values.sortIndex, + values.sortAsc, + chartData, ]); // getting redux state @@ -619,10 +648,29 @@ const DataDocChartComposerComponent: React.FunctionComponent< ); } + const sortDOM = ( + <> + Sort + + + + + + + + ); + const chartTabDOM = ( <> {chartOptionsDOM} {axesDOM} + {sortDOM} ); @@ -739,11 +787,7 @@ const DataDocChartComposerComponent: React.FunctionComponent< dataDOM = (
@@ -757,7 +801,6 @@ const DataDocChartComposerComponent: React.FunctionComponent< {dataSwitch} {hideTableButtonDOM} - {dataDOM}
); @@ -877,12 +920,18 @@ function formValsToMeta(vals: IChartFormValues, meta: IDataChartCellMeta) { for (const [field, val] of Object.entries(vals.xAxis)) { draft.chart.x_axis[field] = val; } - draft.chart.y_axis.stack = vals.stack; + if (vals.sortIndex != null) { + draft.chart.x_axis.sort = { + idx: vals.sortIndex, + asc: vals.sortAsc, + }; + } // Y Axes for (const [field, val] of Object.entries(vals.yAxis)) { draft.chart.y_axis[field] = val; } + draft.chart.y_axis.stack = vals.stack; const seriesObj = {}; if (vals.hiddenSeries.length) { diff --git a/datahub/webapp/const/dataDocChart.ts b/datahub/webapp/const/dataDocChart.ts index 5c593dc10..3d324c21d 100644 --- a/datahub/webapp/const/dataDocChart.ts +++ b/datahub/webapp/const/dataDocChart.ts @@ -72,6 +72,10 @@ export interface IChartAxisMeta { export interface IChartXAxisMeta extends IChartAxisMeta { col_idx: number; + sort: { + idx: number; + asc: boolean; + }; } export interface IChartYAxisMeta extends IChartAxisMeta { @@ -144,6 +148,8 @@ export interface IChartFormValues { xAxis: IChartAxisMeta; xIndex: number; + sortIndex: number | undefined; + sortAsc: boolean; yAxis: IChartAxisMeta; stack: boolean; hiddenSeries: number[]; diff --git a/datahub/webapp/lib/chart/chart-data-processing.ts b/datahub/webapp/lib/chart/chart-data-processing.ts index 12cd57dd4..8ed02261c 100644 --- a/datahub/webapp/lib/chart/chart-data-processing.ts +++ b/datahub/webapp/lib/chart/chart-data-processing.ts @@ -22,7 +22,7 @@ export function processChartJSData( const xAxisIdx = chartMeta.x_axis.col_idx; const seriesNames: string[] = data[0]; - const dataRows = sortTable(data.slice(1), xAxisIdx); + const dataRows = data.slice(1); // hide hidden series const hiddenSeriesIndices = new Set(); diff --git a/datahub/webapp/lib/chart/chart-data-transformation.ts b/datahub/webapp/lib/chart/chart-data-transformation.ts index 482b3538b..e9f65ded3 100644 --- a/datahub/webapp/lib/chart/chart-data-transformation.ts +++ b/datahub/webapp/lib/chart/chart-data-transformation.ts @@ -1,5 +1,6 @@ import { cloneDeep, range } from 'lodash'; import { ChartDataAggType } from 'const/dataDocChart'; +import { sortTable } from './chart-utils'; const emptyCellValue = 'No Value'; @@ -227,6 +228,15 @@ function aggregateData( return aggregatedData; } +export function sortData(data: any[][], idx: number, ascending: boolean) { + const sortIdx = idx ?? (isNaN(data[0][0]) ? undefined : 0); + if (sortIdx != null) { + return sortTable(data, sortIdx, ascending); + } else { + return data; + } +} + export function transformData( data: any[][], isAggregate: boolean = false, @@ -236,16 +246,14 @@ export function transformData( formatValueCols: number[] = [2], aggSeries: { [seriesIdx: number]: ChartDataAggType; - } = {} + } = {}, + sortIdx: number, + sortAsc: boolean = true ) { if (data?.length < 2) { return null; } - if (!isAggregate && !isSwitch) { - return data; - } - let transformedData = cloneDeep(data); if (isAggregate) { @@ -262,5 +270,8 @@ export function transformData( transformedData = switchData(transformedData); } - return transformedData; + return [ + transformedData[0], + ...sortData(transformedData.slice(1), sortIdx, sortAsc), + ]; } diff --git a/datahub/webapp/lib/chart/chart-meta-processing.ts b/datahub/webapp/lib/chart/chart-meta-processing.ts index 229686048..7144e4f4f 100644 --- a/datahub/webapp/lib/chart/chart-meta-processing.ts +++ b/datahub/webapp/lib/chart/chart-meta-processing.ts @@ -63,6 +63,8 @@ export function getDataTransformationOptions(meta: IDataChartCellMeta) { switch: Boolean(transformations.switch), aggSeries, aggType, + sortIndex: meta.chart.x_axis.sort?.idx, + sortAsc: meta.chart.x_axis.sort.asc || true, }; } @@ -106,6 +108,8 @@ export function mapMetaToFormVals( // axes xAxis: getAxisOptions(meta.chart.x_axis), xIndex: meta.chart.x_axis.col_idx, + sortIndex: meta.chart.x_axis.sort?.idx, + sortAsc: meta.chart.x_axis.sort?.asc ?? true, yAxis: getAxisOptions(meta.chart.y_axis), stack: Boolean(meta.chart.y_axis.stack), diff --git a/datahub/webapp/lib/chart/chart-utils.ts b/datahub/webapp/lib/chart/chart-utils.ts index 13481088c..310122be5 100644 --- a/datahub/webapp/lib/chart/chart-utils.ts +++ b/datahub/webapp/lib/chart/chart-utils.ts @@ -48,7 +48,8 @@ export function getDefaultScaleType(value: any): ChartScaleType { export function sortTable( tableRows: any[][], - columnIndex: number = 0 + columnIndex: number = 0, + ascending: boolean = true ): any[][] { // Check if string are numbers, if so use number sort // otherwise use default sort @@ -58,12 +59,24 @@ export function sortTable( } if (!isNaN(tableRows[0][columnIndex] as number)) { - return tableRows.sort( - (a, b) => a[columnIndex] - b[columnIndex] - ) as any[]; + if (ascending) { + return tableRows.sort( + (a, b) => a[columnIndex] - b[columnIndex] + ) as any[]; + } else { + return tableRows.sort( + (a, b) => b[columnIndex] - a[columnIndex] + ) as any[]; + } } - // TODO: do not sort categorical data until sort feature - // return tableRows.sort((a, b) => (a[columnIndex] > b[columnIndex] ? 1 : -1)); - return tableRows; + if (ascending) { + return tableRows.sort((a, b) => + a[columnIndex] > b[columnIndex] ? 1 : -1 + ); + } else { + return tableRows.sort((a, b) => + a[columnIndex] < b[columnIndex] ? 1 : -1 + ); + } } From bf215b854f1a71723dbd5e7ea8309adc5ec04c51 Mon Sep 17 00:00:00 2001 From: Lena Date: Mon, 4 May 2020 11:11:33 -0700 Subject: [PATCH 3/7] update as per comments --- .../DataDocChartCell/DataDocChartCell.tsx | 4 +- .../DataDocChartCell/DataDocChartComposer.tsx | 65 +++++++++---------- datahub/webapp/const/dataDocChart.ts | 2 +- .../lib/chart/chart-data-transformation.ts | 21 ++++-- .../webapp/lib/chart/chart-meta-processing.ts | 3 +- datahub/webapp/lib/chart/chart-utils.ts | 26 +++----- 6 files changed, 60 insertions(+), 61 deletions(-) diff --git a/datahub/webapp/components/DataDocChartCell/DataDocChartCell.tsx b/datahub/webapp/components/DataDocChartCell/DataDocChartCell.tsx index 653858e35..c48151d96 100644 --- a/datahub/webapp/components/DataDocChartCell/DataDocChartCell.tsx +++ b/datahub/webapp/components/DataDocChartCell/DataDocChartCell.tsx @@ -131,6 +131,7 @@ export const DataDocChartCell: React.FunctionComponent = ({ aggSeries, sortIndex, sortAsc, + xAxisIdx, } = getDataTransformationOptions(meta); return transformData( @@ -142,7 +143,8 @@ export const DataDocChartCell: React.FunctionComponent = ({ formatValueCols, aggSeries, sortIndex, - sortAsc + sortAsc, + xAxisIdx ); } } diff --git a/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx b/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx index 2f6db774f..4029beaf0 100644 --- a/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx +++ b/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx @@ -23,7 +23,7 @@ import { colorPalette, colorPaletteNames } from 'const/chartColors'; import { defaultReactSelectStyles } from 'lib/utils/react-select'; import { mapMetaToFormVals } from 'lib/chart/chart-meta-processing'; -import { transformData, sortData } from 'lib/chart/chart-data-transformation'; +import { transformData } from 'lib/chart/chart-data-transformation'; import { useChartSource } from 'hooks/chart/useChartSource'; import { DataDocChart } from './DataDocChart'; @@ -43,7 +43,6 @@ import { Level, LevelItem } from 'ui/Level/Level'; import { SimpleReactSelect } from 'ui/SimpleReactSelect/SimpleReactSelect'; import { getDefaultScaleType } from 'lib/chart/chart-utils'; import { NumberField } from 'ui/FormikField/NumberField'; -import { CheckboxField } from 'ui/FormikField/CheckboxField'; import { ReactSelectField } from 'ui/FormikField/ReactSelectField'; import { FormWrapper } from 'ui/Form/FormWrapper'; import { SimpleField } from 'ui/FormikField/SimpleField'; @@ -111,7 +110,8 @@ const DataDocChartComposerComponent: React.FunctionComponent< values.formatValueCols, values.aggSeries, values.sortIndex, - values.sortAsc + values.sortAsc, + values.xIndex ) : null; }, [ @@ -125,32 +125,16 @@ const DataDocChartComposerComponent: React.FunctionComponent< values.aggSeries, values.sortIndex, values.sortAsc, + values.xIndex, ]); const tableData: any[][] = React.useMemo(() => { if (tableTab === 'original') { - if (values.sortIndex != null && statementResultData) { - return [ - statementResultData[0], - ...sortData( - statementResultData.slice(1), - values.sortIndex, - values.sortAsc - ), - ]; - } else { - return statementResultData; - } + return statementResultData; } else { return chartData; } - }, [ - tableTab, - statementResultData, - values.sortIndex, - values.sortAsc, - chartData, - ]); + }, [tableTab, statementResultData, chartData]); // getting redux state const queryCellOptions = useSelector((state: IStoreState) => { @@ -651,18 +635,29 @@ const DataDocChartComposerComponent: React.FunctionComponent< const sortDOM = ( <> Sort - - - - - - + { + setFieldValue('sortIndex', val); + if (val != null) { + setTableTab('transformed'); + } + }} + /> + ); @@ -770,7 +765,7 @@ const DataDocChartComposerComponent: React.FunctionComponent< let dataDOM: JSX.Element; let dataSwitch: JSX.Element; if (chartData && showTable) { - if (values.aggregate || values.switch) { + if (values.aggregate || values.switch || values.sortIndex != null) { dataSwitch = (
a[columnIndex] - b[columnIndex] - ) as any[]; - } else { - return tableRows.sort( - (a, b) => b[columnIndex] - a[columnIndex] - ) as any[]; - } + return tableRows.sort( + (a, b) => (a[columnIndex] - b[columnIndex]) * reverseMultiplier + ) as any[]; } - if (ascending) { - return tableRows.sort((a, b) => - a[columnIndex] > b[columnIndex] ? 1 : -1 - ); - } else { - return tableRows.sort((a, b) => - a[columnIndex] < b[columnIndex] ? 1 : -1 - ); - } + return tableRows.sort( + (a, b) => (a[columnIndex] > b[columnIndex] ? 1 : -1) * reverseMultiplier + ); } From 29fa3a4b9f7475ce497cac6975de68ee39ab12ca Mon Sep 17 00:00:00 2001 From: Lena Date: Mon, 4 May 2020 12:19:44 -0700 Subject: [PATCH 4/7] update as per comments --- .../webapp/components/AppAdmin/AppAdmin.scss | 1 + .../DataDocChartCell/DataDocChartComposer.tsx | 14 ++++------ .../lib/chart/chart-data-transformation.ts | 28 ++++++++++++++----- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/datahub/webapp/components/AppAdmin/AppAdmin.scss b/datahub/webapp/components/AppAdmin/AppAdmin.scss index 397082df9..aa0f637c8 100644 --- a/datahub/webapp/components/AppAdmin/AppAdmin.scss +++ b/datahub/webapp/components/AppAdmin/AppAdmin.scss @@ -64,6 +64,7 @@ flex-grow: 1; text-align: left; overflow-y: hidden; + overflow-x: hidden; .AdminForm-button-box { display: flex; justify-content: center; diff --git a/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx b/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx index 4029beaf0..00454b099 100644 --- a/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx +++ b/datahub/webapp/components/DataDocChartCell/DataDocChartComposer.tsx @@ -128,14 +128,6 @@ const DataDocChartComposerComponent: React.FunctionComponent< values.xIndex, ]); - const tableData: any[][] = React.useMemo(() => { - if (tableTab === 'original') { - return statementResultData; - } else { - return chartData; - } - }, [tableTab, statementResultData, chartData]); - // getting redux state const queryCellOptions = useSelector((state: IStoreState) => { const cellList = state.dataDoc.dataDocById[dataDocId].cells; @@ -782,7 +774,11 @@ const DataDocChartComposerComponent: React.FunctionComponent< dataDOM = (
diff --git a/datahub/webapp/lib/chart/chart-data-transformation.ts b/datahub/webapp/lib/chart/chart-data-transformation.ts index 538dae31f..475f36987 100644 --- a/datahub/webapp/lib/chart/chart-data-transformation.ts +++ b/datahub/webapp/lib/chart/chart-data-transformation.ts @@ -1,6 +1,6 @@ import { cloneDeep, range } from 'lodash'; import { ChartDataAggType } from 'const/dataDocChart'; -import { sortTable } from './chart-utils'; +import { sortTable, getValueDataType } from './chart-utils'; const emptyCellValue = 'No Value'; @@ -228,17 +228,26 @@ function aggregateData( return aggregatedData; } -export function sortData( +export function sortTableWithDefaultIdx( data: any[][], idx: number, ascending: boolean, xIdx: number ) { - const sortIdx = idx ?? (isNaN(data[0][xIdx]) ? undefined : 0); - if (sortIdx != null) { - return sortTable(data, sortIdx, ascending); + if (idx != null) { + // if idx is provided then call sortTable as is + return sortTable(data, idx, ascending); } else { - return data; + // if idx is not provided then default idx to the xIndex of the table + // only sort if the column type for that index is either 'date' 'datetime' or 'number' + if ( + getValueDataType(data[0][xIdx]) === 'string' || + getValueDataType(data[0][xIdx]) === null + ) { + return data; + } else { + return sortTable(data, xIdx, ascending); + } } } @@ -283,6 +292,11 @@ export function transformData( return [ transformedData[0], - ...sortData(transformedData.slice(1), sortIdx, sortAsc, xAxisIdx), + ...sortTableWithDefaultIdx( + transformedData.slice(1), + sortIdx, + sortAsc, + xAxisIdx + ), ]; } From 8e1ee5dd0b772e01b62882530286b3757f5f79b6 Mon Sep 17 00:00:00 2001 From: Lena Date: Mon, 4 May 2020 12:28:42 -0700 Subject: [PATCH 5/7] update --- datahub/webapp/lib/chart/chart-data-transformation.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/datahub/webapp/lib/chart/chart-data-transformation.ts b/datahub/webapp/lib/chart/chart-data-transformation.ts index 475f36987..6ad042367 100644 --- a/datahub/webapp/lib/chart/chart-data-transformation.ts +++ b/datahub/webapp/lib/chart/chart-data-transformation.ts @@ -240,10 +240,7 @@ export function sortTableWithDefaultIdx( } else { // if idx is not provided then default idx to the xIndex of the table // only sort if the column type for that index is either 'date' 'datetime' or 'number' - if ( - getValueDataType(data[0][xIdx]) === 'string' || - getValueDataType(data[0][xIdx]) === null - ) { + if (['string', null].includes(getValueDataType(data[0][xIdx]))) { return data; } else { return sortTable(data, xIdx, ascending); From 44f3c9e1f6580c04d3fb24bb8f0a7860044462e4 Mon Sep 17 00:00:00 2001 From: Lena Date: Mon, 4 May 2020 12:30:20 -0700 Subject: [PATCH 6/7] update --- datahub/webapp/lib/chart/chart-data-transformation.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datahub/webapp/lib/chart/chart-data-transformation.ts b/datahub/webapp/lib/chart/chart-data-transformation.ts index 6ad042367..4b95730d1 100644 --- a/datahub/webapp/lib/chart/chart-data-transformation.ts +++ b/datahub/webapp/lib/chart/chart-data-transformation.ts @@ -240,7 +240,8 @@ export function sortTableWithDefaultIdx( } else { // if idx is not provided then default idx to the xIndex of the table // only sort if the column type for that index is either 'date' 'datetime' or 'number' - if (['string', null].includes(getValueDataType(data[0][xIdx]))) { + const valType = getValueDataType(data[0][xIdx]); + if (['string', null].includes(valType)) { return data; } else { return sortTable(data, xIdx, ascending); From 222ffed324fc5e102fc2841d4b6eef936bdaa9b8 Mon Sep 17 00:00:00 2001 From: Lena Date: Mon, 4 May 2020 12:32:12 -0700 Subject: [PATCH 7/7] update --- datahub/webapp/lib/chart/chart-data-transformation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub/webapp/lib/chart/chart-data-transformation.ts b/datahub/webapp/lib/chart/chart-data-transformation.ts index 4b95730d1..effdaad72 100644 --- a/datahub/webapp/lib/chart/chart-data-transformation.ts +++ b/datahub/webapp/lib/chart/chart-data-transformation.ts @@ -241,7 +241,7 @@ export function sortTableWithDefaultIdx( // if idx is not provided then default idx to the xIndex of the table // only sort if the column type for that index is either 'date' 'datetime' or 'number' const valType = getValueDataType(data[0][xIdx]); - if (['string', null].includes(valType)) { + if (valType === 'string' || valType === null) { return data; } else { return sortTable(data, xIdx, ascending);