From 39e6c6eac648b8703c9ab88ba35207f23d40cffc Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Mon, 22 Aug 2022 11:34:25 +0800 Subject: [PATCH 01/17] =?UTF-8?q?feat[interfaces]:=E6=A0=87=E8=AE=B0?= =?UTF-8?q?=E6=89=A9=E5=B1=95=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/rath-client/src/interfaces.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/rath-client/src/interfaces.ts b/packages/rath-client/src/interfaces.ts index 26cfd852..840579a5 100644 --- a/packages/rath-client/src/interfaces.ts +++ b/packages/rath-client/src/interfaces.ts @@ -13,12 +13,24 @@ export interface IRow { export type IGeoRole = 'longitude' | 'latitude' | 'none'; +/** Detailed information of a extended field. */ +interface IFieldExtInfoBase { + /** Field id of fields that this field infered from. */ + extFrom: string[]; + /** The identifier of the data-extension operation. */ + extOpt: string; + /** Additional information of the specified extension operation */ + extInfo: any; +} + interface IFieldBase { fid: string; name?: string; analyticType: IAnalyticType; semanticType: ISemanticType; geoRole: IGeoRole; + /** detailed information of field extension operations. defined only if this field is extended */ + extInfo?: IFieldExtInfoBase; } export interface IRawField extends IFieldBase { disable?: boolean; @@ -35,6 +47,8 @@ export interface IMuteFieldBase { semanticType: 'nominal' | 'temporal' | 'ordinal' | 'quantitative' | '?'; disable?: boolean | '?'; geoRole: IGeoRole | '?'; + /** detailed information of field extension operations. Non-null if this field is extended */ + extInfo?: IFieldExtInfoBase; } export interface IFieldMeta extends IFieldBase { From 61734b969343d4b7509fc2c75a7b8768c505847d Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Tue, 23 Aug 2022 13:29:17 +0800 Subject: [PATCH 02/17] =?UTF-8?q?feat:=E5=B1=95=E5=BC=80=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E5=AD=97=E6=AE=B5(expandDateTime)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../rath-client/public/locales/en-US.json | 3 + .../rath-client/public/locales/zh-CN.json | 3 + packages/rath-client/src/dev/services.ts | 36 +++ .../src/dev/workers/dateTimeExpand.worker.js | 21 ++ .../src/dev/workers/engine/dateTimeExpand.ts | 212 ++++++++++++++++++ .../src/pages/dataSource/index.tsx | 8 + .../rath-client/src/store/dataSourceStore.ts | 29 +++ 7 files changed, 312 insertions(+) create mode 100644 packages/rath-client/src/dev/services.ts create mode 100644 packages/rath-client/src/dev/workers/dateTimeExpand.worker.js create mode 100644 packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index afae27b3..f0bc675f 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -167,6 +167,9 @@ }, "extend": { "title": "Extend Features" + }, + "expandDateTime": { + "title": "Expand DateTime Fields" } }, "meta": { diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index d57bcb70..18b10714 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -167,6 +167,9 @@ }, "extend": { "title": "扩展维度" + }, + "expandDateTime": { + "title": "扩展时间字段" } }, "meta": { diff --git a/packages/rath-client/src/dev/services.ts b/packages/rath-client/src/dev/services.ts new file mode 100644 index 00000000..48e8e770 --- /dev/null +++ b/packages/rath-client/src/dev/services.ts @@ -0,0 +1,36 @@ +import { IAnalyticType, IRow, ISemanticType } from "visual-insights"; +import { IGeoRole, IMuteFieldBase, IRawField } from "rath-client/src/interfaces"; +import { inferAnalyticType, inferAnalyticTypeFromSemanticType, inferSemanticType } from "rath-client/src/utils"; +import { workerService } from "rath-client/src/service"; +/* eslint import/no-webpack-loader-syntax:0 */ +// @ts-ignore +// eslint-disable-next-line +import ExpandDateTimeWorker from './workers/dateTimeExpand.worker.js?worker'; +import { dateTimeExpand } from './workers/engine/dateTimeExpand' + +interface ExpandDateTimeProps { + dataSource: IRow[]; + fields: IRawField[]; +} +export async function expandDateTimeService (props: ExpandDateTimeProps): Promise { + if (process.env.NODE_ENV === 'development') { + let res = dateTimeExpand(props) as ExpandDateTimeProps + return res + } + else { + try{ + const worker = new ExpandDateTimeWorker() + const result = await workerService(worker, props) + worker.terminate(); + if (result.success) { + return result.data + } + else { + throw new Error(`[ExpandDateTimeWorker]: ${result.message}`) + } + } catch (error) { + console.error(`[ExpandDateTimeWorker]: ${error}`) + throw error; + } + } +} \ No newline at end of file diff --git a/packages/rath-client/src/dev/workers/dateTimeExpand.worker.js b/packages/rath-client/src/dev/workers/dateTimeExpand.worker.js new file mode 100644 index 00000000..165575d7 --- /dev/null +++ b/packages/rath-client/src/dev/workers/dateTimeExpand.worker.js @@ -0,0 +1,21 @@ +/* eslint no-restricted-globals: 0 */ +import { dateTimeExpand } from "./engine/dateTimeExpand"; +import { timer } from '../../workers/timer'; + +const expandService = e => { + try { + const { fields, dataSource } = e.data; + const res = dateTimeExpand({ fields, dataSource }) + self.postMessage({ + success: true, + data: res + }) + } catch (error) { + self.postMessage({ + success: false, + message: error.toString() + }) + } +} + +self.addEventListener('message', timer(expandService), false) \ No newline at end of file diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts new file mode 100644 index 00000000..599b2974 --- /dev/null +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -0,0 +1,212 @@ +import { IAnalyticType, IRow, ISemanticType } from "visual-insights"; +import { IGeoRole, IMuteFieldBase, IRawField } from "rath-client/src/interfaces"; +import { field } from "vega"; + +class Knowable { + value: T; + known: boolean; + constructor(type: { new(): T; }, val: T|undefined = undefined) { + if (val === undefined) { + this.value = new type(); + this.known = false + } + else { + this.value = val + this.known = true + } + } + set(val: T) { + this.known = true + this.value = val + } +} +interface DateTimeInfo { + utime: Knowable; // unix timestamp + $y: Knowable; // Year + $M: Knowable; // Month - starts from 1 + $D: Knowable; // Day - starts from 1 + $L: Knowable; // Locale + $W: Knowable; // Week + $H: Knowable; // Hour + $m: Knowable; // minute + $s: Knowable; // second + $ms: Knowable; // milliseconds +} +const dateTimeDict = new Map([ + ['utime', 'utime'], + ['$y', 'year'], + ['$M', 'month'], + ['$D', 'date'], + ['$L', 'locale'], + ['$W', 'weekday'], + ['$H', 'hour'], + ['$m', 'min'], + ['$s', 'sec'], + ['$ms', 'ms'] +]) + +const REGEX_PARSE = /^(\d{2,4})[-/](\d{1,2})?[-/]?(\d{0,2})[Tt\s]*(\d{1,2})?:?(\d{1,2})?:?(\d{1,2})?[.:]?(\d+)?$/ +function Date2Info(date: Date): DateTimeInfo { + return { + utime: new Knowable(Number, date.getTime() ), + $y: new Knowable(Number, date.getUTCFullYear() ), + $M: new Knowable(Number, date.getUTCMonth()+1 ), + $D: new Knowable(Number, date.getUTCDate() ), + $W: new Knowable(Number, date.getUTCDay() ), + $H: new Knowable(Number, date.getUTCHours() ), + $m: new Knowable(Number, date.getUTCMinutes() ), + $s: new Knowable(Number, date.getUTCSeconds() ), + $ms: new Knowable(Number, date.getUTCMilliseconds()), + $L: new Knowable(String) + } +} +function parseDateTime(dateTime: string): DateTimeInfo { + if (dateTime === null) return {} as DateTimeInfo + if (dateTime === undefined) { + let date = new Date() + return Date2Info(date) + } + if (!/Z$/i.test(dateTime)) { + const match = dateTime.match(REGEX_PARSE) + if (match) { + let $y, $M, $D, $H, $m, $s, $ms + if (match[1]) $y = parseInt(match[1]) + else throw new Error("[parseDateTime]: Missing 'Year' in dateTime string") + if (match[2]) $M = parseInt(match[2]) + else $M = undefined + if (match[3]) $D = parseInt(match[3]) + else $D = undefined + if (match[4]) $H = parseInt(match[4]) + else $H = undefined + if (match[5]) $m = parseInt(match[5]) + else $m = undefined + if (match[6]) $s = parseInt(match[6]) + else $s = undefined + if (match[7]) $ms = parseInt(match[7].substring(0, 3)) + else $ms = undefined + let dateTime = new Date(Date.UTC($y, ($M || 1) - 1, $D || 1, $H || 0, $m || 0, $s || 0, $ms || 0)) + let utime = dateTime.getTime(), $W; + if (match[1] && match[2] && match[3]) { + $W = dateTime.getUTCDay() + } + return { + utime: new Knowable(Number, utime), + $y: new Knowable(Number, $y), + $M: new Knowable(Number, $M), + $D: new Knowable(Number, $D), + $W: new Knowable(Number, $W), + $H: new Knowable(Number, $H), + $m: new Knowable(Number, $m), + $s: new Knowable(Number, $s), + $ms: new Knowable(Number, $ms), + $L: new Knowable(String, undefined) + } as DateTimeInfo + } + } + // Polyfill + return Date2Info(new Date(dateTime)) +} + +interface DateTimeInfoArray { + utime: Knowable[]; // unix timestamp + $y: Knowable[]; // Year + $M?: Knowable[]; // Month - starts from 0 + $D?: Knowable[]; // Day - starts from 1 + $W?: Knowable[]; // Week + $H?: Knowable[]; // Hour + $m?: Knowable[]; // minute + $s?: Knowable[]; // second + $ms?: Knowable[]; // milliseconds + $L?: Knowable[]; // Locale +} +const UnknownDateTimeInfo: DateTimeInfo = { + utime: new Knowable(Number), + $y: new Knowable(Number), + $M: new Knowable(Number), + $D: new Knowable(Number), + $W: new Knowable(Number), + $H: new Knowable(Number), + $m: new Knowable(Number), + $s: new Knowable(Number), + $ms: new Knowable(Number), + $L: new Knowable(String) +} + +type InfoArrayType = keyof DateTimeInfoArray +type InfoType = keyof DateTimeInfo +function parseDateTimeArray(dateTime: string[]): DateTimeInfoArray { + // TODO: Polyfills: 中文格式等 + let infoArray = { + utime: new Array>(dateTime.length), + $y: new Array>(dateTime.length) + } as DateTimeInfoArray + for (let i = 0;i < dateTime.length; ++i) { + let info = parseDateTime(dateTime[i]) + Object.keys(info).forEach(key => { + let infoKey = key as InfoType, infoArrayKey = key as InfoArrayType + if (info[infoKey].known) { + if (infoArray[infoArrayKey] === undefined) { + let array: Knowable[] = new Array>(dateTime.length) + array.fill(UnknownDateTimeInfo[infoKey], 0, i) + array[i] = info[infoKey] + infoArray[infoArrayKey] = array + } + else { + (infoArray[infoArrayKey] as Knowable[]).fill(info[infoKey], i, i+1) + } + } + }) + } + return infoArray +} + +export function dateTimeExpand(props: { dataSource: IRow[]; fields: IMuteFieldBase[] }) + : { dataSource: IRow[]; fields: IMuteFieldBase[] } { + const { dataSource, fields } = props; + + let extFields: IMuteFieldBase[] = [] + let fieldIds = new Set(fields.map(f => (f.extInfo && f.extInfo?.extInfo === "dateTimeExpand") ? f.fid : '')) + fields.forEach(field => { + extFields.push(field) + if (field.semanticType === 'temporal' && !fieldIds.has(field.fid)) { + let dateTime = dataSource.map(item => item[field.fid]) + let moment: DateTimeInfoArray = parseDateTimeArray(dateTime) + Object.keys(moment).forEach(key => { + let extField: IMuteFieldBase[] = [{ + fid: `${field.fid}_${key}`, + name: `${field.name}.${dateTimeDict.get(key)}`, + analyticType: 'dimension', + semanticType: 'ordinal', + geoRole: 'none', + extInfo: { + extFrom: [field.fid], + extOpt: 'dateTimeExpand', + extInfo: `${key}.value` + } + }, + { + fid: `${field.fid}_${key}_known`, + name: `${field.name}.${dateTimeDict.get(key)}.known`, + analyticType: 'dimension', + semanticType: 'nominal', + geoRole: 'none', + extInfo: { + extFrom: [field.fid], + extOpt: 'dateTimeExpand', + extInfo: `${key}.known` + } + }] + extFields.push(...extField) + let infoArray = moment[key as InfoArrayType] as Knowable[] + for (let i = 0; i < dataSource.length; ++i) { + dataSource[i][extField[0].fid] = infoArray[i].value + dataSource[i][extField[1].fid] = infoArray[i].known ? 1 : 0 + } + }) + } + }) + return { + dataSource: dataSource, + fields: extFields + } +} diff --git a/packages/rath-client/src/pages/dataSource/index.tsx b/packages/rath-client/src/pages/dataSource/index.tsx index c28ef016..c86da7cc 100644 --- a/packages/rath-client/src/pages/dataSource/index.tsx +++ b/packages/rath-client/src/pages/dataSource/index.tsx @@ -254,6 +254,14 @@ const DataSourceBoard: React.FC = (props) => { dataSourceStore.extendData(); }} /> + { + dataSourceStore.expandDateTime(); + }} + iconProps={{ iconName: 'CalendarMirrored' }} + /> {intl.get('dataSource.recordCount', { count: cleanedData.length })}
diff --git a/packages/rath-client/src/store/dataSourceStore.ts b/packages/rath-client/src/store/dataSourceStore.ts index 672a929f..d1994d23 100644 --- a/packages/rath-client/src/store/dataSourceStore.ts +++ b/packages/rath-client/src/store/dataSourceStore.ts @@ -8,6 +8,8 @@ import { RATH_INDEX_COLUMN_KEY } from "../constants"; import { IDataPreviewMode, IDatasetBase, IFieldMeta, IMuteFieldBase, IRawField, IRow, IFilter, CleanMethod, IDataPrepProgressTag } from "../interfaces"; import { getQuantiles } from "../pages/dataSource/utils"; import { cleanDataService, extendDataService, filterDataService, getFieldsSummaryService, inferMetaService } from "../service"; +import { expandDateTimeService } from "../dev/services"; +// import { expandDateTimeService } from "../service"; import { findRathSafeColumnIndex } from "../utils"; import { fieldSummary2fieldMeta } from "../utils/transform"; @@ -400,6 +402,7 @@ export class DataSourceStore { } public async extendData () { + // TODO: IRawField增加了extInfo?: IFieldExtInfoBase属性，此处应在新增字段的时候补充详细信息 try { const { fields, cleanedData } = this; const res = await extendDataService({ @@ -444,4 +447,30 @@ export class DataSourceStore { }) } } + + /** + * Expand all temporal fields to (year, month, date, weekday, hour, minute, second, millisecond). + * @depends this.fields, this.cleanedDate + * @effects this.rawData, this.mutFields + */ + public async expandDateTime () { + try { + let { fields, cleanedData } = this; + fields = fields.map(f => toJS(f)) + const res = await expandDateTimeService({ + dataSource: cleanedData, + fields + }) + runInAction(() => { + this.rawData = res.dataSource; + this.mutFields = res.fields + }) + } catch (error) { + notify({ + title: 'Expand DateTime API Error', + type: 'error', + content: `[extension]${error}` + }) + } + } } From 6e6be490f940a7da7a01d4326100b5bae25353c0 Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Tue, 23 Aug 2022 15:21:52 +0800 Subject: [PATCH 03/17] perf[dateTimeExpand]:temporally disable undefined-value marks --- .../src/dev/workers/engine/dateTimeExpand.ts | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index 599b2974..9729acfb 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -172,7 +172,7 @@ export function dateTimeExpand(props: { dataSource: IRow[]; fields: IMuteFieldBa let dateTime = dataSource.map(item => item[field.fid]) let moment: DateTimeInfoArray = parseDateTimeArray(dateTime) Object.keys(moment).forEach(key => { - let extField: IMuteFieldBase[] = [{ + let extField: IMuteFieldBase = { fid: `${field.fid}_${key}`, name: `${field.name}.${dateTimeDict.get(key)}`, analyticType: 'dimension', @@ -183,24 +183,11 @@ export function dateTimeExpand(props: { dataSource: IRow[]; fields: IMuteFieldBa extOpt: 'dateTimeExpand', extInfo: `${key}.value` } - }, - { - fid: `${field.fid}_${key}_known`, - name: `${field.name}.${dateTimeDict.get(key)}.known`, - analyticType: 'dimension', - semanticType: 'nominal', - geoRole: 'none', - extInfo: { - extFrom: [field.fid], - extOpt: 'dateTimeExpand', - extInfo: `${key}.known` - } - }] - extFields.push(...extField) + } + extFields.push(extField) let infoArray = moment[key as InfoArrayType] as Knowable[] for (let i = 0; i < dataSource.length; ++i) { - dataSource[i][extField[0].fid] = infoArray[i].value - dataSource[i][extField[1].fid] = infoArray[i].known ? 1 : 0 + dataSource[i][extField.fid] = infoArray[i].value } }) } From 51a5874271f5dfd6b267638684175b0734454d55 Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Tue, 23 Aug 2022 16:07:37 +0800 Subject: [PATCH 04/17] style:code clean-up & formatting --- packages/rath-client/src/dev/services.ts | 9 +-- .../src/dev/workers/engine/dateTimeExpand.ts | 79 +++++++++---------- packages/rath-client/src/interfaces.ts | 2 +- .../rath-client/src/store/dataSourceStore.ts | 2 +- 4 files changed, 45 insertions(+), 47 deletions(-) diff --git a/packages/rath-client/src/dev/services.ts b/packages/rath-client/src/dev/services.ts index 48e8e770..a07df884 100644 --- a/packages/rath-client/src/dev/services.ts +++ b/packages/rath-client/src/dev/services.ts @@ -1,6 +1,5 @@ -import { IAnalyticType, IRow, ISemanticType } from "visual-insights"; -import { IGeoRole, IMuteFieldBase, IRawField } from "rath-client/src/interfaces"; -import { inferAnalyticType, inferAnalyticTypeFromSemanticType, inferSemanticType } from "rath-client/src/utils"; +import { IRow } from "visual-insights"; +import { IRawField } from "rath-client/src/interfaces"; import { workerService } from "rath-client/src/service"; /* eslint import/no-webpack-loader-syntax:0 */ // @ts-ignore @@ -12,13 +11,13 @@ interface ExpandDateTimeProps { dataSource: IRow[]; fields: IRawField[]; } -export async function expandDateTimeService (props: ExpandDateTimeProps): Promise { +export async function expandDateTimeService(props: ExpandDateTimeProps): Promise { if (process.env.NODE_ENV === 'development') { let res = dateTimeExpand(props) as ExpandDateTimeProps return res } else { - try{ + try { const worker = new ExpandDateTimeWorker() const result = await workerService(worker, props) worker.terminate(); diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index 9729acfb..dd82595f 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -1,11 +1,10 @@ -import { IAnalyticType, IRow, ISemanticType } from "visual-insights"; -import { IGeoRole, IMuteFieldBase, IRawField } from "rath-client/src/interfaces"; -import { field } from "vega"; +import { IRow } from "visual-insights"; +import { IMuteFieldBase } from "rath-client/src/interfaces"; class Knowable { value: T; known: boolean; - constructor(type: { new(): T; }, val: T|undefined = undefined) { + constructor(type: { new(): T; }, val: T | undefined = undefined) { if (val === undefined) { this.value = new type(); this.known = false @@ -22,14 +21,14 @@ class Knowable { } interface DateTimeInfo { utime: Knowable; // unix timestamp - $y: Knowable; // Year - $M: Knowable; // Month - starts from 1 - $D: Knowable; // Day - starts from 1 - $L: Knowable; // Locale - $W: Knowable; // Week - $H: Knowable; // Hour - $m: Knowable; // minute - $s: Knowable; // second + $y: Knowable; // Year + $M: Knowable; // Month - starts from 1 + $D: Knowable; // Day - starts from 1 + $L: Knowable; // Locale + $W: Knowable; // Week + $H: Knowable; // Hour + $m: Knowable; // minute + $s: Knowable; // second $ms: Knowable; // milliseconds } const dateTimeDict = new Map([ @@ -48,16 +47,16 @@ const dateTimeDict = new Map([ const REGEX_PARSE = /^(\d{2,4})[-/](\d{1,2})?[-/]?(\d{0,2})[Tt\s]*(\d{1,2})?:?(\d{1,2})?:?(\d{1,2})?[.:]?(\d+)?$/ function Date2Info(date: Date): DateTimeInfo { return { - utime: new Knowable(Number, date.getTime() ), - $y: new Knowable(Number, date.getUTCFullYear() ), - $M: new Knowable(Number, date.getUTCMonth()+1 ), - $D: new Knowable(Number, date.getUTCDate() ), - $W: new Knowable(Number, date.getUTCDay() ), - $H: new Knowable(Number, date.getUTCHours() ), - $m: new Knowable(Number, date.getUTCMinutes() ), - $s: new Knowable(Number, date.getUTCSeconds() ), + utime: new Knowable(Number, date.getTime()), + $y: new Knowable(Number, date.getUTCFullYear()), + $M: new Knowable(Number, date.getUTCMonth() + 1), + $D: new Knowable(Number, date.getUTCDate()), + $W: new Knowable(Number, date.getUTCDay()), + $H: new Knowable(Number, date.getUTCHours()), + $m: new Knowable(Number, date.getUTCMinutes()), + $s: new Knowable(Number, date.getUTCSeconds()), $ms: new Knowable(Number, date.getUTCMilliseconds()), - $L: new Knowable(String) + $L: new Knowable(String) } } function parseDateTime(dateTime: string): DateTimeInfo { @@ -91,15 +90,15 @@ function parseDateTime(dateTime: string): DateTimeInfo { } return { utime: new Knowable(Number, utime), - $y: new Knowable(Number, $y), - $M: new Knowable(Number, $M), - $D: new Knowable(Number, $D), - $W: new Knowable(Number, $W), - $H: new Knowable(Number, $H), - $m: new Knowable(Number, $m), - $s: new Knowable(Number, $s), + $y: new Knowable(Number, $y), + $M: new Knowable(Number, $M), + $D: new Knowable(Number, $D), + $W: new Knowable(Number, $W), + $H: new Knowable(Number, $H), + $m: new Knowable(Number, $m), + $s: new Knowable(Number, $s), $ms: new Knowable(Number, $ms), - $L: new Knowable(String, undefined) + $L: new Knowable(String, undefined) } as DateTimeInfo } } @@ -109,15 +108,15 @@ function parseDateTime(dateTime: string): DateTimeInfo { interface DateTimeInfoArray { utime: Knowable[]; // unix timestamp - $y: Knowable[]; // Year - $M?: Knowable[]; // Month - starts from 0 - $D?: Knowable[]; // Day - starts from 1 - $W?: Knowable[]; // Week - $H?: Knowable[]; // Hour - $m?: Knowable[]; // minute - $s?: Knowable[]; // second + $y: Knowable[]; // Year + $M?: Knowable[]; // Month - starts from 0 + $D?: Knowable[]; // Day - starts from 1 + $W?: Knowable[]; // Week + $H?: Knowable[]; // Hour + $m?: Knowable[]; // minute + $s?: Knowable[]; // second $ms?: Knowable[]; // milliseconds - $L?: Knowable[]; // Locale + $L?: Knowable[]; // Locale } const UnknownDateTimeInfo: DateTimeInfo = { utime: new Knowable(Number), @@ -140,7 +139,7 @@ function parseDateTimeArray(dateTime: string[]): DateTimeInfoArray { utime: new Array>(dateTime.length), $y: new Array>(dateTime.length) } as DateTimeInfoArray - for (let i = 0;i < dateTime.length; ++i) { + for (let i = 0; i < dateTime.length; ++i) { let info = parseDateTime(dateTime[i]) Object.keys(info).forEach(key => { let infoKey = key as InfoType, infoArrayKey = key as InfoArrayType @@ -152,9 +151,9 @@ function parseDateTimeArray(dateTime: string[]): DateTimeInfoArray { infoArray[infoArrayKey] = array } else { - (infoArray[infoArrayKey] as Knowable[]).fill(info[infoKey], i, i+1) + (infoArray[infoArrayKey] as Knowable[]).fill(info[infoKey], i, i + 1) } - } + } }) } return infoArray diff --git a/packages/rath-client/src/interfaces.ts b/packages/rath-client/src/interfaces.ts index 840579a5..d504f6a1 100644 --- a/packages/rath-client/src/interfaces.ts +++ b/packages/rath-client/src/interfaces.ts @@ -16,7 +16,7 @@ export type IGeoRole = 'longitude' | 'latitude' | 'none'; /** Detailed information of a extended field. */ interface IFieldExtInfoBase { /** Field id of fields that this field infered from. */ - extFrom: string[]; + extFrom: string[]; /** The identifier of the data-extension operation. */ extOpt: string; /** Additional information of the specified extension operation */ diff --git a/packages/rath-client/src/store/dataSourceStore.ts b/packages/rath-client/src/store/dataSourceStore.ts index d1994d23..f80be2b2 100644 --- a/packages/rath-client/src/store/dataSourceStore.ts +++ b/packages/rath-client/src/store/dataSourceStore.ts @@ -453,7 +453,7 @@ export class DataSourceStore { * @depends this.fields, this.cleanedDate * @effects this.rawData, this.mutFields */ - public async expandDateTime () { + public async expandDateTime() { try { let { fields, cleanedData } = this; fields = fields.map(f => toJS(f)) From 364a4b5fe0aa5057cfb732bbb79cd6deabd2a4f1 Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Tue, 23 Aug 2022 17:54:26 +0800 Subject: [PATCH 05/17] fix:dateTimeExpand fallback --- .../src/dev/workers/engine/dateTimeExpand.ts | 87 ++++++++++--------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index dd82595f..055b7f58 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -60,50 +60,55 @@ function Date2Info(date: Date): DateTimeInfo { } } function parseDateTime(dateTime: string): DateTimeInfo { - if (dateTime === null) return {} as DateTimeInfo - if (dateTime === undefined) { - let date = new Date() - return Date2Info(date) - } - if (!/Z$/i.test(dateTime)) { - const match = dateTime.match(REGEX_PARSE) - if (match) { - let $y, $M, $D, $H, $m, $s, $ms - if (match[1]) $y = parseInt(match[1]) - else throw new Error("[parseDateTime]: Missing 'Year' in dateTime string") - if (match[2]) $M = parseInt(match[2]) - else $M = undefined - if (match[3]) $D = parseInt(match[3]) - else $D = undefined - if (match[4]) $H = parseInt(match[4]) - else $H = undefined - if (match[5]) $m = parseInt(match[5]) - else $m = undefined - if (match[6]) $s = parseInt(match[6]) - else $s = undefined - if (match[7]) $ms = parseInt(match[7].substring(0, 3)) - else $ms = undefined - let dateTime = new Date(Date.UTC($y, ($M || 1) - 1, $D || 1, $H || 0, $m || 0, $s || 0, $ms || 0)) - let utime = dateTime.getTime(), $W; - if (match[1] && match[2] && match[3]) { - $W = dateTime.getUTCDay() + try{ + if (dateTime === null) return UnknownDateTimeInfo; + if (dateTime === undefined) { + let date = new Date() + return Date2Info(date) + } + if (!/Z$/i.test(dateTime)) { + const match = dateTime.match(REGEX_PARSE) + if (match) { + let $y, $M, $D, $H, $m, $s, $ms + if (match[1]) $y = parseInt(match[1]) + else throw new Error("[parseDateTime]: Missing 'Year' in dateTime string") + if (match[2]) $M = parseInt(match[2]) + else $M = undefined + if (match[3]) $D = parseInt(match[3]) + else $D = undefined + if (match[4]) $H = parseInt(match[4]) + else $H = undefined + if (match[5]) $m = parseInt(match[5]) + else $m = undefined + if (match[6]) $s = parseInt(match[6]) + else $s = undefined + if (match[7]) $ms = parseInt(match[7].substring(0, 3)) + else $ms = undefined + let dateTime = new Date(Date.UTC($y, ($M || 1) - 1, $D || 1, $H || 0, $m || 0, $s || 0, $ms || 0)) + let utime = dateTime.getTime(), $W; + if (match[1] && match[2] && match[3]) { + $W = dateTime.getUTCDay() + } + return { + utime: new Knowable(Number, utime), + $y: new Knowable(Number, $y), + $M: new Knowable(Number, $M), + $D: new Knowable(Number, $D), + $W: new Knowable(Number, $W), + $H: new Knowable(Number, $H), + $m: new Knowable(Number, $m), + $s: new Knowable(Number, $s), + $ms: new Knowable(Number, $ms), + $L: new Knowable(String, undefined) + } as DateTimeInfo } - return { - utime: new Knowable(Number, utime), - $y: new Knowable(Number, $y), - $M: new Knowable(Number, $M), - $D: new Knowable(Number, $D), - $W: new Knowable(Number, $W), - $H: new Knowable(Number, $H), - $m: new Knowable(Number, $m), - $s: new Knowable(Number, $s), - $ms: new Knowable(Number, $ms), - $L: new Knowable(String, undefined) - } as DateTimeInfo } + // Polyfill + return Date2Info(new Date(dateTime)) + } + catch(error) { + return UnknownDateTimeInfo } - // Polyfill - return Date2Info(new Date(dateTime)) } interface DateTimeInfoArray { From efb165fb63172331d502c3e1b0e425254fc88ab9 Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Tue, 23 Aug 2022 22:10:30 +0800 Subject: [PATCH 06/17] refactor:remove 'Knowable' related code --- .../src/dev/workers/engine/dateTimeExpand.ts | 137 +++++++----------- 1 file changed, 50 insertions(+), 87 deletions(-) diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index 055b7f58..76ea34e4 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -1,35 +1,17 @@ import { IRow } from "visual-insights"; import { IMuteFieldBase } from "rath-client/src/interfaces"; -class Knowable { - value: T; - known: boolean; - constructor(type: { new(): T; }, val: T | undefined = undefined) { - if (val === undefined) { - this.value = new type(); - this.known = false - } - else { - this.value = val - this.known = true - } - } - set(val: T) { - this.known = true - this.value = val - } -} interface DateTimeInfo { - utime: Knowable; // unix timestamp - $y: Knowable; // Year - $M: Knowable; // Month - starts from 1 - $D: Knowable; // Day - starts from 1 - $L: Knowable; // Locale - $W: Knowable; // Week - $H: Knowable; // Hour - $m: Knowable; // minute - $s: Knowable; // second - $ms: Knowable; // milliseconds + utime?: number; // unix timestamp + $y?: number; // Year + $M?: number; // Month - starts from 1 + $D?: number; // Day - starts from 1 + $L?: string; // Locale + $W?: number; // Week + $H?: number; // Hour + $m?: number; // minute + $s?: number; // second + $ms?: number; // milliseconds } const dateTimeDict = new Map([ ['utime', 'utime'], @@ -47,20 +29,20 @@ const dateTimeDict = new Map([ const REGEX_PARSE = /^(\d{2,4})[-/](\d{1,2})?[-/]?(\d{0,2})[Tt\s]*(\d{1,2})?:?(\d{1,2})?:?(\d{1,2})?[.:]?(\d+)?$/ function Date2Info(date: Date): DateTimeInfo { return { - utime: new Knowable(Number, date.getTime()), - $y: new Knowable(Number, date.getUTCFullYear()), - $M: new Knowable(Number, date.getUTCMonth() + 1), - $D: new Knowable(Number, date.getUTCDate()), - $W: new Knowable(Number, date.getUTCDay()), - $H: new Knowable(Number, date.getUTCHours()), - $m: new Knowable(Number, date.getUTCMinutes()), - $s: new Knowable(Number, date.getUTCSeconds()), - $ms: new Knowable(Number, date.getUTCMilliseconds()), - $L: new Knowable(String) + utime: date.getTime(), + $y: date.getUTCFullYear(), + $M: date.getUTCMonth() + 1, + $D: date.getUTCDate(), + $W: date.getUTCDay(), + $H: date.getUTCHours(), + $m: date.getUTCMinutes(), + $s: date.getUTCSeconds(), + $ms: date.getUTCMilliseconds(), + $L: "" } } function parseDateTime(dateTime: string): DateTimeInfo { - try{ + try { if (dateTime === null) return UnknownDateTimeInfo; if (dateTime === undefined) { let date = new Date() @@ -69,7 +51,7 @@ function parseDateTime(dateTime: string): DateTimeInfo { if (!/Z$/i.test(dateTime)) { const match = dateTime.match(REGEX_PARSE) if (match) { - let $y, $M, $D, $H, $m, $s, $ms + let $y, $M, $D, $W, $H, $m, $s, $ms if (match[1]) $y = parseInt(match[1]) else throw new Error("[parseDateTime]: Missing 'Year' in dateTime string") if (match[2]) $M = parseInt(match[2]) @@ -85,21 +67,20 @@ function parseDateTime(dateTime: string): DateTimeInfo { if (match[7]) $ms = parseInt(match[7].substring(0, 3)) else $ms = undefined let dateTime = new Date(Date.UTC($y, ($M || 1) - 1, $D || 1, $H || 0, $m || 0, $s || 0, $ms || 0)) - let utime = dateTime.getTime(), $W; + let utime = dateTime.getTime(); if (match[1] && match[2] && match[3]) { $W = dateTime.getUTCDay() } return { - utime: new Knowable(Number, utime), - $y: new Knowable(Number, $y), - $M: new Knowable(Number, $M), - $D: new Knowable(Number, $D), - $W: new Knowable(Number, $W), - $H: new Knowable(Number, $H), - $m: new Knowable(Number, $m), - $s: new Knowable(Number, $s), - $ms: new Knowable(Number, $ms), - $L: new Knowable(String, undefined) + utime: utime, + $y: $y, + $M: $M, + $D: $D, + $W: $W, + $H: $H, + $m: $m, + $s: $s, + $ms: $ms } as DateTimeInfo } } @@ -107,57 +88,39 @@ function parseDateTime(dateTime: string): DateTimeInfo { return Date2Info(new Date(dateTime)) } catch(error) { + console.warn(error) return UnknownDateTimeInfo } } interface DateTimeInfoArray { - utime: Knowable[]; // unix timestamp - $y: Knowable[]; // Year - $M?: Knowable[]; // Month - starts from 0 - $D?: Knowable[]; // Day - starts from 1 - $W?: Knowable[]; // Week - $H?: Knowable[]; // Hour - $m?: Knowable[]; // minute - $s?: Knowable[]; // second - $ms?: Knowable[]; // milliseconds - $L?: Knowable[]; // Locale -} -const UnknownDateTimeInfo: DateTimeInfo = { - utime: new Knowable(Number), - $y: new Knowable(Number), - $M: new Knowable(Number), - $D: new Knowable(Number), - $W: new Knowable(Number), - $H: new Knowable(Number), - $m: new Knowable(Number), - $s: new Knowable(Number), - $ms: new Knowable(Number), - $L: new Knowable(String) + utime?: any[]; // unix timestamp + $y?: any[]; // Year + $M?: any[]; // Month - starts from 0 + $D?: any[]; // Day - starts from 1 + $W?: any[]; // Week + $H?: any[]; // Hour + $m?: any[]; // minute + $s?: any[]; // second + $ms?: any[]; // milliseconds + $L?: any[]; // Locale } +const UnknownDateTimeInfo: DateTimeInfo = {} type InfoArrayType = keyof DateTimeInfoArray type InfoType = keyof DateTimeInfo function parseDateTimeArray(dateTime: string[]): DateTimeInfoArray { // TODO: Polyfills: 中文格式等 - let infoArray = { - utime: new Array>(dateTime.length), - $y: new Array>(dateTime.length) - } as DateTimeInfoArray + let infoArray = {} as DateTimeInfoArray for (let i = 0; i < dateTime.length; ++i) { let info = parseDateTime(dateTime[i]) Object.keys(info).forEach(key => { let infoKey = key as InfoType, infoArrayKey = key as InfoArrayType - if (info[infoKey].known) { + if (info[infoKey] !== undefined && info[infoKey] !== "") { if (infoArray[infoArrayKey] === undefined) { - let array: Knowable[] = new Array>(dateTime.length) - array.fill(UnknownDateTimeInfo[infoKey], 0, i) - array[i] = info[infoKey] - infoArray[infoArrayKey] = array - } - else { - (infoArray[infoArrayKey] as Knowable[]).fill(info[infoKey], i, i + 1) + infoArray[infoArrayKey] = new Array(dateTime.length) } + (infoArray[infoArrayKey] as any[])[i] = info[infoKey] } }) } @@ -189,9 +152,9 @@ export function dateTimeExpand(props: { dataSource: IRow[]; fields: IMuteFieldBa } } extFields.push(extField) - let infoArray = moment[key as InfoArrayType] as Knowable[] + let infoArray = moment[key as InfoArrayType] as any[] for (let i = 0; i < dataSource.length; ++i) { - dataSource[i][extField.fid] = infoArray[i].value + dataSource[i][extField.fid] = infoArray[i] } }) } From ee5d0962c16c5de738e05998f5101564c2161572 Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Wed, 24 Aug 2022 22:40:01 +0800 Subject: [PATCH 07/17] =?UTF-8?q?fix:=E6=89=A9=E5=B1=95=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E5=90=8E=E4=BF=9D=E7=95=99disable=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/rath-client/src/store/dataSourceStore.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/rath-client/src/store/dataSourceStore.ts b/packages/rath-client/src/store/dataSourceStore.ts index f80be2b2..f223af42 100644 --- a/packages/rath-client/src/store/dataSourceStore.ts +++ b/packages/rath-client/src/store/dataSourceStore.ts @@ -455,11 +455,11 @@ export class DataSourceStore { */ public async expandDateTime() { try { - let { fields, cleanedData } = this; - fields = fields.map(f => toJS(f)) + let { mutFields, cleanedData } = this; + mutFields = mutFields.map(f => toJS(f)) const res = await expandDateTimeService({ dataSource: cleanedData, - fields + fields: mutFields }) runInAction(() => { this.rawData = res.dataSource; From a2217f410a3a2536301227b8dfcc7cd4b93f7cab Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Fri, 26 Aug 2022 00:19:33 +0800 Subject: [PATCH 08/17] =?UTF-8?q?refactor:=E9=87=8D=E6=9E=84=E6=AD=A3?= =?UTF-8?q?=E5=88=99=E5=8C=B9=E9=85=8D=20=E6=96=B9=E4=BE=BF=E6=89=A9?= =?UTF-8?q?=E5=B1=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/rath-client/src/dev/services.ts | 19 +- .../src/dev/workers/engine/dateTimeExpand.ts | 244 +++++++++++++++--- .../src/dev/workers/engine/synAnalyzer.ts | 29 +++ 3 files changed, 246 insertions(+), 46 deletions(-) create mode 100644 packages/rath-client/src/dev/workers/engine/synAnalyzer.ts diff --git a/packages/rath-client/src/dev/services.ts b/packages/rath-client/src/dev/services.ts index a07df884..45081d15 100644 --- a/packages/rath-client/src/dev/services.ts +++ b/packages/rath-client/src/dev/services.ts @@ -5,14 +5,27 @@ import { workerService } from "rath-client/src/service"; // @ts-ignore // eslint-disable-next-line import ExpandDateTimeWorker from './workers/dateTimeExpand.worker.js?worker'; -import { dateTimeExpand } from './workers/engine/dateTimeExpand' +import { dateTimeExpand, doTest } from './workers/engine/dateTimeExpand' interface ExpandDateTimeProps { dataSource: IRow[]; fields: IRawField[]; } -export async function expandDateTimeService(props: ExpandDateTimeProps): Promise { - if (process.env.NODE_ENV === 'development') { +function checkExpandEnv(): string { + if (typeof window === 'object') { + const url = new URL(window.location.href).searchParams.get('expand'); + if(url) return url + else return '' + } + if (process.env.EXPAND_ENV) return process.env.EXPAND_ENV + else return '' +} +const DebugEnv = checkExpandEnv(); + + +export async function expandDateTimeService (props: ExpandDateTimeProps): Promise { + if (DebugEnv === 'debug') { + doTest() let res = dateTimeExpand(props) as ExpandDateTimeProps return res } diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index 76ea34e4..387eafea 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -1,7 +1,8 @@ import { IRow } from "visual-insights"; import { IMuteFieldBase } from "rath-client/src/interfaces"; +import { LexAnalyzer, LexAnalyzerItem, SynAnalyzerRule } from "./synAnalyzer" -interface DateTimeInfo { +interface IDateTimeInfo { utime?: number; // unix timestamp $y?: number; // Year $M?: number; // Month - starts from 1 @@ -13,6 +14,38 @@ interface DateTimeInfo { $s?: number; // second $ms?: number; // milliseconds } +class DateTimeInfo implements IDateTimeInfo { + utime?: number; // unix timestamp + $y?: number; // Year + $M?: number; // Month - starts from 1 + $D?: number; // Day - starts from 1 + $L?: string; // Locale + $W?: number; // Week + $H?: number; // Hour + $m?: number; // minute + $s?: number; // second + $ms?: number; // milliseconds + constructor(y?: string, M?: string, D?: string, H?: string, m?: string, s?: string, ms?: string, L?: string) { + if (y) this.$y = parseInt(y) + else throw new Error("[parseDateTime]: Missing 'Year' in dateTime string") + if (M) this.$M = parseInt(M) + if (D) this.$D = parseInt(D) + if (H) this.$H = parseInt(H) + if (m) this.$m = parseInt(m) + if (s) this.$s = parseInt(s) + if (ms) this.$ms = parseInt(ms) + this.$L = L + let dateTime = new Date(Date.UTC(this.$y, (this.$M || 1) - 1, this.$D || 1, this.$H || 0, this.$m || 0, this.$s || 0, this.$ms || 0)) + this.utime = dateTime.getTime(); + if (y && M && D) { + this.$W = dateTime.getUTCDay() + } + } +} +/** parameter locations of DateTimeInfo constructor */ +enum DateTimeParamPos { + year = 0, month, date, hour, min, sec, ms +} const dateTimeDict = new Map([ ['utime', 'utime'], ['$y', 'year'], @@ -25,8 +58,125 @@ const dateTimeDict = new Map([ ['$s', 'sec'], ['$ms', 'ms'] ]) +export class DateTimeLexAnalyzer implements LexAnalyzer { + static MONTH_NAME_LIST = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + static MONTH_ABBR_LIST = ['Jan', 'Feb', 'March', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sept', 'Oct', 'Nov', 'Dec'] + static MONTH_MAP = new Map( + this.MONTH_NAME_LIST.map<[string, string]>((s, i) => [s, `${i + 1}`]) + .concat(this.MONTH_ABBR_LIST.map((s, i) => [s, `${i + 1}`])) + ) + static TODAY = new Date() + static CUR_YEAR = this.TODAY.getUTCFullYear() + symbols = { + YEAR: Symbol.for('lex_year'), + YEAR2: Symbol.for('lex_year2'), + MONTH: Symbol.for('lex_month'), + MONTH_NAME: Symbol.for('lex_month_name'), + MONTH_ABBR: Symbol.for('lex_month_abbr'), + DATE: Symbol.for('lex_date'), + HOUR: Symbol.for('lex_hour'), + MIN: Symbol.for('lex_min'), + SEC: Symbol.for('lex_sec'), + MS: Symbol.for('lex_ms') + } + items: {[x: symbol]: [LexAnalyzerItem, DateTimeParamPos]} = { + // [symbol]: [ LexAnalyzerItem, data index in DateTime constructor param list ] + [this.symbols.YEAR]: [ + new LexAnalyzerItem('[1-2][0-9]{3}'), DateTimeParamPos.year + ], + [this.symbols.YEAR2]: [ + new LexAnalyzerItem('[0-9]{2}', (p?: string) => p && ((parseInt("20" + p) <= DateTimeLexAnalyzer.CUR_YEAR) ? "20" + p : "19" + p)), DateTimeParamPos.year + ], + [this.symbols.MONTH]: [ + new LexAnalyzerItem('0?[1-9]|1[0-2]'), DateTimeParamPos.month + ], + [this.symbols.MONTH_NAME]: [ + new LexAnalyzerItem(DateTimeLexAnalyzer.MONTH_NAME_LIST.join('|'), (p?: string) => p && DateTimeLexAnalyzer.MONTH_MAP.get(p)), DateTimeParamPos.month + ], + [this.symbols.MONTH_ABBR]: [ + new LexAnalyzerItem(DateTimeLexAnalyzer.MONTH_ABBR_LIST.join('|'), (p?: string) => p && DateTimeLexAnalyzer.MONTH_MAP.get(p)), DateTimeParamPos.month + ], + [this.symbols.DATE]: [ + new LexAnalyzerItem('(?:0?[1-9])|(?:[1-2][0-9])|(?:3[0-1])'), DateTimeParamPos.date + ], + [this.symbols.HOUR]: [ + new LexAnalyzerItem('[0-1]?[0-9]|(?:2[0-4])'), DateTimeParamPos.hour + ], + [this.symbols.MIN]: [ + new LexAnalyzerItem('[-:][0-5]?[0-9]', (p?: string) => p?.slice(1)), DateTimeParamPos.min + ], + [this.symbols.SEC]: [ + new LexAnalyzerItem('[-:][0-5]?[0-9]', (p?: string) => p?.slice(1)), DateTimeParamPos.sec + ], + [this.symbols.MS]: [ + new LexAnalyzerItem('[\\.:-][0-9]{3}', (p?: string) => p && ((p[0] === '.') ? p + '000' : p).substring(1, 4)), DateTimeParamPos.ms + ] + } + trans(regRes: RegExpExecArray, type: symbol[]): [string | undefined, DateTimeParamPos][] { + if (regRes.length !== type.length + 1) throw new Error("[DateTimeLexAnalyzer.trans]: different length") + let res = new Array<[string | undefined, DateTimeParamPos]>(type.length) + for (let i = 0;i < type.length; ++i) { + res[i] = [ this.items[type[i]][0].trans(regRes[i+1]), this.items[type[i]][1] ] + } + return res + } +} +export class DateTimeSynAnalyzer { + lex = new DateTimeLexAnalyzer() + rules: Array + constructor() { + let s = this.lex.symbols + this.rules = [ + new SynAnalyzerRule(this.lex, + [s.YEAR, s.MONTH, s.DATE, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})[-/]?(${r[1]})[-/]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // YYYY-MM-DD [HH[:mm[:ss[.ms]]]] + new SynAnalyzerRule(this.lex, + [s.YEAR2, s.MONTH, s.DATE, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})[-/]?(${r[1]})[-/]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // YY-MM-DD [HH[:mm[:ss[.ms]]]] + new SynAnalyzerRule(this.lex, + [s.MONTH, s.DATE, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})[-/](${r[1]})[-/](${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // MM-DD-YYYY + new SynAnalyzerRule(this.lex, + [s.MONTH, s.DATE, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})[-/](${r[1]})[-/](${r[2]})(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // MM-DD-YY + new SynAnalyzerRule(this.lex, + [s.MONTH_NAME, s.DATE, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})[A-Za-z\\.,\\s]*(${r[1]})[A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // Mo-DD-YYYY + new SynAnalyzerRule(this.lex, + [s.MONTH_NAME, s.DATE, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})[A-Za-z\\.,\\s]*(${r[1]})[A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // Mo-DD-YY + new SynAnalyzerRule(this.lex, + [s.MONTH_ABBR, s.DATE, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})[A-Za-z\\.,\\s]*(${r[1]})[A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // Mo.-DD-YYYY + new SynAnalyzerRule(this.lex, + [s.MONTH_ABBR, s.DATE, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})[A-Za-z\\.,\\s]*(${r[1]})[A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ) // Mo.-DD-YY + ] + } +} + +let analyzer = new DateTimeSynAnalyzer() +export function parseReg(value: string, reg_id: number): DateTimeInfo { + if (!(Number.isInteger(reg_id) && reg_id < analyzer.rules.length)) throw new Error("reg_id error") + let parse_res = analyzer.rules[reg_id].exec(value) + if (parse_res) { + let res = analyzer.lex.trans(parse_res, analyzer.rules[reg_id].symbols) + let infoArray = new Array(7) + for (let i = 0; i < res.length; ++i) infoArray[res[i][1]] = res[i][0]; + // console.log(analyzer.rules[reg_id].reg, reg_id, infoArray) + return new DateTimeInfo(...infoArray) + } + return {} as DateTimeInfo +} -const REGEX_PARSE = /^(\d{2,4})[-/](\d{1,2})?[-/]?(\d{0,2})[Tt\s]*(\d{1,2})?:?(\d{1,2})?:?(\d{1,2})?[.:]?(\d+)?$/ function Date2Info(date: Date): DateTimeInfo { return { utime: date.getTime(), @@ -41,49 +191,17 @@ function Date2Info(date: Date): DateTimeInfo { $L: "" } } -function parseDateTime(dateTime: string): DateTimeInfo { + +function parseDateTime(dateTime: string, reg_id?: number): DateTimeInfo { try { + if (reg_id !== undefined) return parseReg(dateTime, reg_id) if (dateTime === null) return UnknownDateTimeInfo; if (dateTime === undefined) { let date = new Date() return Date2Info(date) } - if (!/Z$/i.test(dateTime)) { - const match = dateTime.match(REGEX_PARSE) - if (match) { - let $y, $M, $D, $W, $H, $m, $s, $ms - if (match[1]) $y = parseInt(match[1]) - else throw new Error("[parseDateTime]: Missing 'Year' in dateTime string") - if (match[2]) $M = parseInt(match[2]) - else $M = undefined - if (match[3]) $D = parseInt(match[3]) - else $D = undefined - if (match[4]) $H = parseInt(match[4]) - else $H = undefined - if (match[5]) $m = parseInt(match[5]) - else $m = undefined - if (match[6]) $s = parseInt(match[6]) - else $s = undefined - if (match[7]) $ms = parseInt(match[7].substring(0, 3)) - else $ms = undefined - let dateTime = new Date(Date.UTC($y, ($M || 1) - 1, $D || 1, $H || 0, $m || 0, $s || 0, $ms || 0)) - let utime = dateTime.getTime(); - if (match[1] && match[2] && match[3]) { - $W = dateTime.getUTCDay() - } - return { - utime: utime, - $y: $y, - $M: $M, - $D: $D, - $W: $W, - $H: $H, - $m: $m, - $s: $s, - $ms: $ms - } as DateTimeInfo - } - } + // if (!/Z$/i.test(dateTime)) { + // } // Polyfill return Date2Info(new Date(dateTime)) } @@ -112,8 +230,16 @@ type InfoType = keyof DateTimeInfo function parseDateTimeArray(dateTime: string[]): DateTimeInfoArray { // TODO: Polyfills: 中文格式等 let infoArray = {} as DateTimeInfoArray + let reg_id: number | undefined, max_cnt = 0; + for (let i = 0; i < analyzer.rules.length; ++i) { + let cnt = dateTime.map(d => analyzer.rules[i].test(d) ? 1 : 0).reduce((a, b) => a + b) + if (cnt > max_cnt) { + reg_id = i + max_cnt = cnt + } + } for (let i = 0; i < dateTime.length; ++i) { - let info = parseDateTime(dateTime[i]) + let info = parseDateTime(dateTime[i], max_cnt > 0 ? reg_id : undefined) Object.keys(info).forEach(key => { let infoKey = key as InfoType, infoArrayKey = key as InfoArrayType if (info[infoKey] !== undefined && info[infoKey] !== "") { @@ -132,9 +258,11 @@ export function dateTimeExpand(props: { dataSource: IRow[]; fields: IMuteFieldBa const { dataSource, fields } = props; let extFields: IMuteFieldBase[] = [] - let fieldIds = new Set(fields.map(f => (f.extInfo && f.extInfo?.extInfo === "dateTimeExpand") ? f.fid : '')) - fields.forEach(field => { + let fieldIds = new Set(fields.map(f => (f.extInfo && f.extInfo?.extOpt === "dateTimeExpand") ? f.extInfo?.extFrom[0] : '')) + for (let i = 0;i < fields.length; ++i) { + const field = fields[i] extFields.push(field) + if (field.disable) continue if (field.semanticType === 'temporal' && !fieldIds.has(field.fid)) { let dateTime = dataSource.map(item => item[field.fid]) let moment: DateTimeInfoArray = parseDateTimeArray(dateTime) @@ -158,9 +286,39 @@ export function dateTimeExpand(props: { dataSource: IRow[]; fields: IMuteFieldBa } }) } - }) + } return { dataSource: dataSource, fields: extFields } } + +export function dateTimeExpandTest(testCase: Array) { + console.log(parseDateTimeArray(testCase)) + for (let i = 0;i < analyzer.rules.length;++i) { + let cnt = 0 + for (let s of testCase) { + let res = analyzer.rules[i].exec(s) + if (res) { + console.log(i, s, res); + cnt += 1; + } + } + console.log(i, cnt, analyzer.rules[i].reg) + } + console.log(parseDateTimeArray(testCase)) +} + +export function doTest() { + if (typeof window === 'object') { + (window as any).parseDateTimeArray = parseDateTimeArray; + (window as any).dateTimeExpandTest = dateTimeExpandTest + } + let testCases: Array> = [ + ["1998-04-09"], + ["98-05-09", "02-04-09", "00-04-12"], + ["980409 12:12:13.98", "220409 12:12:13.98"], + ["Apr. 8th, 2014 13:13"] + ] + for (let c of testCases) dateTimeExpandTest(c) +} \ No newline at end of file diff --git a/packages/rath-client/src/dev/workers/engine/synAnalyzer.ts b/packages/rath-client/src/dev/workers/engine/synAnalyzer.ts new file mode 100644 index 00000000..4415f9da --- /dev/null +++ b/packages/rath-client/src/dev/workers/engine/synAnalyzer.ts @@ -0,0 +1,29 @@ + +export class LexAnalyzerItem { + regExp: string; + trans: (arg0?: string) => string | undefined; + constructor(regExp: string, transform?: (arg0?: string) => string | undefined) { + this.regExp = regExp + this.trans = transform || ((p?: string) => p) + } +} +export interface LexAnalyzer { + symbols: { [s: string]: symbol }; + items: {[x: symbol]: [LexAnalyzerItem, T]}; + trans: (regRes: RegExpExecArray, type: symbol[]) => [string | undefined, T][]; +} +export class SynAnalyzerRule { + reg: RegExp + /** + * construct a SynAnalyzer rule + * @param lex: LexAnalyzer + * @param symbols symbol of entities this rule involves (in order) + * @param getReg: (regExps: string[]) => string 接收symbols中每个元素对应的正则表达式, 拼接出该规则的完整表达式 + */ + constructor(public lex: LexAnalyzer, public symbols: symbol[], getReg: (regExps: string[]) => string) { + let regs = symbols.map((s: symbol) => lex.items[s][0].regExp) + this.reg = new RegExp(getReg(regs), 'i') + } + test(s: string): boolean { return this.reg.test(s) } + exec(s: string): RegExpExecArray | null { return this.reg.exec(s) } +} From dda02e431b9b55b21a44d5c706016f39032a5a2a Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Fri, 26 Aug 2022 17:33:24 +0800 Subject: [PATCH 09/17] fix:correct DateTime rules; test:add test cases --- packages/rath-client/src/dev/services.ts | 15 ++------- .../src/dev/workers/engine/checkExpandEnv.ts | 12 +++++++ .../src/dev/workers/engine/dateTimeExpand.ts | 31 +++++++++++-------- .../src/dev/workers/engine/synAnalyzer.ts | 3 ++ 4 files changed, 36 insertions(+), 25 deletions(-) create mode 100644 packages/rath-client/src/dev/workers/engine/checkExpandEnv.ts diff --git a/packages/rath-client/src/dev/services.ts b/packages/rath-client/src/dev/services.ts index 45081d15..c133fa8b 100644 --- a/packages/rath-client/src/dev/services.ts +++ b/packages/rath-client/src/dev/services.ts @@ -6,25 +6,16 @@ import { workerService } from "rath-client/src/service"; // eslint-disable-next-line import ExpandDateTimeWorker from './workers/dateTimeExpand.worker.js?worker'; import { dateTimeExpand, doTest } from './workers/engine/dateTimeExpand' +import { checkExpandEnv } from './workers/engine/checkExpandEnv' interface ExpandDateTimeProps { dataSource: IRow[]; fields: IRawField[]; } -function checkExpandEnv(): string { - if (typeof window === 'object') { - const url = new URL(window.location.href).searchParams.get('expand'); - if(url) return url - else return '' - } - if (process.env.EXPAND_ENV) return process.env.EXPAND_ENV - else return '' -} -const DebugEnv = checkExpandEnv(); - +const ExpandEnv = checkExpandEnv(); export async function expandDateTimeService (props: ExpandDateTimeProps): Promise { - if (DebugEnv === 'debug') { + if (ExpandEnv === 'debug') { doTest() let res = dateTimeExpand(props) as ExpandDateTimeProps return res diff --git a/packages/rath-client/src/dev/workers/engine/checkExpandEnv.ts b/packages/rath-client/src/dev/workers/engine/checkExpandEnv.ts new file mode 100644 index 00000000..544a0b68 --- /dev/null +++ b/packages/rath-client/src/dev/workers/engine/checkExpandEnv.ts @@ -0,0 +1,12 @@ +export function checkExpandEnv(): string { + if (typeof window === 'object') { + const url = new URL(window.location.href).searchParams.get('expand'); + if(url) { + (window as any).ExpandEnv = url + return url + } + else return '' + } + if (process.env.EXPAND_ENV) return process.env.EXPAND_ENV + else return '' +} \ No newline at end of file diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index 387eafea..032a8ca0 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -1,6 +1,7 @@ import { IRow } from "visual-insights"; import { IMuteFieldBase } from "rath-client/src/interfaces"; import { LexAnalyzer, LexAnalyzerItem, SynAnalyzerRule } from "./synAnalyzer" +import { checkExpandEnv } from "./checkExpandEnv"; interface IDateTimeInfo { utime?: number; // unix timestamp @@ -109,7 +110,7 @@ export class DateTimeLexAnalyzer implements LexAnalyzer { new LexAnalyzerItem('[-:][0-5]?[0-9]', (p?: string) => p?.slice(1)), DateTimeParamPos.sec ], [this.symbols.MS]: [ - new LexAnalyzerItem('[\\.:-][0-9]{3}', (p?: string) => p && ((p[0] === '.') ? p + '000' : p).substring(1, 4)), DateTimeParamPos.ms + new LexAnalyzerItem('[\\.:-][0-9]{0,3}', (p?: string) => p && ((p[0] === '.') ? p + '000' : p).substring(1, 4)), DateTimeParamPos.ms ] } trans(regRes: RegExpExecArray, type: symbol[]): [string | undefined, DateTimeParamPos][] { @@ -129,19 +130,19 @@ export class DateTimeSynAnalyzer { this.rules = [ new SynAnalyzerRule(this.lex, [s.YEAR, s.MONTH, s.DATE, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/]?(${r[1]})[-/]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s]?(${r[1]})[-/\\s]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // YYYY-MM-DD [HH[:mm[:ss[.ms]]]] new SynAnalyzerRule(this.lex, [s.YEAR2, s.MONTH, s.DATE, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/]?(${r[1]})[-/]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s]?(${r[1]})[-/\\s]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // YY-MM-DD [HH[:mm[:ss[.ms]]]] new SynAnalyzerRule(this.lex, [s.MONTH, s.DATE, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/](${r[1]})[-/](${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s]?(${r[1]})[-/\\s]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // MM-DD-YYYY new SynAnalyzerRule(this.lex, [s.MONTH, s.DATE, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/](${r[1]})[-/](${r[2]})(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s]?(${r[1]})[-/\\s]?(${r[2]})(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // MM-DD-YY new SynAnalyzerRule(this.lex, [s.MONTH_NAME, s.DATE, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], @@ -229,6 +230,7 @@ type InfoArrayType = keyof DateTimeInfoArray type InfoType = keyof DateTimeInfo function parseDateTimeArray(dateTime: string[]): DateTimeInfoArray { // TODO: Polyfills: 中文格式等 + // TODO: assume the same dateTime format or support different format in one column let infoArray = {} as DateTimeInfoArray let reg_id: number | undefined, max_cnt = 0; for (let i = 0; i < analyzer.rules.length; ++i) { @@ -300,25 +302,28 @@ export function dateTimeExpandTest(testCase: Array) { for (let s of testCase) { let res = analyzer.rules[i].exec(s) if (res) { - console.log(i, s, res); + console.log(i, s, analyzer.rules[i].trans(res)); cnt += 1; } } console.log(i, cnt, analyzer.rules[i].reg) } - console.log(parseDateTimeArray(testCase)) +} + +if (typeof window === 'object' && checkExpandEnv().toLowerCase() === 'debug') { + (window as any).parseDateTimeArray = parseDateTimeArray; + (window as any).dateTimeExpandTest = dateTimeExpandTest } export function doTest() { - if (typeof window === 'object') { - (window as any).parseDateTimeArray = parseDateTimeArray; - (window as any).dateTimeExpandTest = dateTimeExpandTest - } let testCases: Array> = [ ["1998-04-09"], + ["19980409"], ["98-05-09", "02-04-09", "00-04-12"], - ["980409 12:12:13.98", "220409 12:12:13.98"], - ["Apr. 8th, 2014 13:13"] + ["980409 12:12:13.98", "220409 12:12:13.198"], + ["Apr. 8th, 2014 13:13"], + ["1998/4/9"], + ["040202", "040502", "040802", "042002", "043102"] ] for (let c of testCases) dateTimeExpandTest(c) } \ No newline at end of file diff --git a/packages/rath-client/src/dev/workers/engine/synAnalyzer.ts b/packages/rath-client/src/dev/workers/engine/synAnalyzer.ts index 4415f9da..eb78c547 100644 --- a/packages/rath-client/src/dev/workers/engine/synAnalyzer.ts +++ b/packages/rath-client/src/dev/workers/engine/synAnalyzer.ts @@ -26,4 +26,7 @@ export class SynAnalyzerRule { } test(s: string): boolean { return this.reg.test(s) } exec(s: string): RegExpExecArray | null { return this.reg.exec(s) } + trans(regRes: RegExpExecArray): [string | undefined, any][] { + return this.lex.trans(regRes, this.symbols) + } } From 8c3a5e51f65fda59de1bfb04a2c48356e3adb72c Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Sat, 27 Aug 2022 17:25:02 +0800 Subject: [PATCH 10/17] test:add dateTimeExpand test cases --- .../src/dev/workers/engine/dateTimeExpand.ts | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index 032a8ca0..842cf734 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -72,9 +72,11 @@ export class DateTimeLexAnalyzer implements LexAnalyzer { YEAR: Symbol.for('lex_year'), YEAR2: Symbol.for('lex_year2'), MONTH: Symbol.for('lex_month'), + MONTH2: Symbol.for('lex_month2'), MONTH_NAME: Symbol.for('lex_month_name'), MONTH_ABBR: Symbol.for('lex_month_abbr'), DATE: Symbol.for('lex_date'), + DATE2: Symbol.for('lex_date2'), HOUR: Symbol.for('lex_hour'), MIN: Symbol.for('lex_min'), SEC: Symbol.for('lex_sec'), @@ -89,7 +91,10 @@ export class DateTimeLexAnalyzer implements LexAnalyzer { new LexAnalyzerItem('[0-9]{2}', (p?: string) => p && ((parseInt("20" + p) <= DateTimeLexAnalyzer.CUR_YEAR) ? "20" + p : "19" + p)), DateTimeParamPos.year ], [this.symbols.MONTH]: [ - new LexAnalyzerItem('0?[1-9]|1[0-2]'), DateTimeParamPos.month + new LexAnalyzerItem('(?:0?[1-9])|(?:1[0-2])'), DateTimeParamPos.month + ], + [this.symbols.MONTH2]: [ + new LexAnalyzerItem('(?:0[1-9])|(?:1[0-2])'), DateTimeParamPos.month ], [this.symbols.MONTH_NAME]: [ new LexAnalyzerItem(DateTimeLexAnalyzer.MONTH_NAME_LIST.join('|'), (p?: string) => p && DateTimeLexAnalyzer.MONTH_MAP.get(p)), DateTimeParamPos.month @@ -100,8 +105,11 @@ export class DateTimeLexAnalyzer implements LexAnalyzer { [this.symbols.DATE]: [ new LexAnalyzerItem('(?:0?[1-9])|(?:[1-2][0-9])|(?:3[0-1])'), DateTimeParamPos.date ], + [this.symbols.DATE2]: [ + new LexAnalyzerItem('(?:0[1-9])|(?:[1-2][0-9])|(?:3[0-1])'), DateTimeParamPos.date + ], [this.symbols.HOUR]: [ - new LexAnalyzerItem('[0-1]?[0-9]|(?:2[0-4])'), DateTimeParamPos.hour + new LexAnalyzerItem('(?:[0-1]?[0-9])|(?:2[0-4])'), DateTimeParamPos.hour ], [this.symbols.MIN]: [ new LexAnalyzerItem('[-:][0-5]?[0-9]', (p?: string) => p?.slice(1)), DateTimeParamPos.min @@ -130,35 +138,51 @@ export class DateTimeSynAnalyzer { this.rules = [ new SynAnalyzerRule(this.lex, [s.YEAR, s.MONTH, s.DATE, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/\\s]?(${r[1]})[-/\\s]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s](${r[1]})(?:[-/\\s](${r[2]}))?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // YYYY-MM-DD [HH[:mm[:ss[.ms]]]] + new SynAnalyzerRule(this.lex, + [s.YEAR, s.MONTH2, s.DATE2, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})(${r[1]})(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // YYYYMMDD [HH[:mm[:ss[.ms]]]] new SynAnalyzerRule(this.lex, [s.YEAR2, s.MONTH, s.DATE, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/\\s]?(${r[1]})[-/\\s]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s](${r[1]})(?:[-/\\s](${r[2]}))?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // YY-MM-DD [HH[:mm[:ss[.ms]]]] + new SynAnalyzerRule(this.lex, + [s.YEAR2, s.MONTH2, s.DATE2, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})(${r[1]})(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // YYMMDD [HH[:mm[:ss[.ms]]]] new SynAnalyzerRule(this.lex, [s.MONTH, s.DATE, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/\\s]?(${r[1]})[-/\\s]?(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s](${r[1]})(?:[-/\\s](${r[2]}))?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // MM-DD-YYYY + new SynAnalyzerRule(this.lex, + [s.MONTH2, s.DATE2, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})(${r[1]})(${r[2]})?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // MMDDYYYY new SynAnalyzerRule(this.lex, [s.MONTH, s.DATE, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/\\s]?(${r[1]})[-/\\s]?(${r[2]})(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s](${r[1]})[-/\\s](${r[2]})(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // MM-DD-YY + new SynAnalyzerRule(this.lex, + [s.MONTH2, s.DATE2, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], + (r: string[]) => `^(${r[0]})(${r[1]})(${r[2]})(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + ), // MMDDYY new SynAnalyzerRule(this.lex, [s.MONTH_NAME, s.DATE, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[A-Za-z\\.,\\s]*(${r[1]})[A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\.,\\s]*(${r[1]})[-/A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // Mo-DD-YYYY new SynAnalyzerRule(this.lex, [s.MONTH_NAME, s.DATE, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[A-Za-z\\.,\\s]*(${r[1]})[A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\.,\\s]*(${r[1]})[-/A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // Mo-DD-YY new SynAnalyzerRule(this.lex, [s.MONTH_ABBR, s.DATE, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[A-Za-z\\.,\\s]*(${r[1]})[A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\.,\\s]*(${r[1]})[-/A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // Mo.-DD-YYYY new SynAnalyzerRule(this.lex, [s.MONTH_ABBR, s.DATE, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[A-Za-z\\.,\\s]*(${r[1]})[A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\.,\\s]*(${r[1]})[-/A-Za-z\\s\\.,]*(${r[2]})(?:[\\.,\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ) // Mo.-DD-YY ] } @@ -240,6 +264,7 @@ function parseDateTimeArray(dateTime: string[]): DateTimeInfoArray { max_cnt = cnt } } + // TODO: 推荐多种不同选择 for (let i = 0; i < dateTime.length; ++i) { let info = parseDateTime(dateTime[i], max_cnt > 0 ? reg_id : undefined) Object.keys(info).forEach(key => { @@ -320,10 +345,15 @@ export function doTest() { ["1998-04-09"], ["19980409"], ["98-05-09", "02-04-09", "00-04-12"], + ["980409", "920409", "000412", "400912"], + ["200924", "200831", "200128"], ["980409 12:12:13.98", "220409 12:12:13.198"], + ["980409 12:12:13:98", "220409 12:12:13:198"], ["Apr. 8th, 2014 13:13"], ["1998/4/9"], - ["040202", "040502", "040802", "042002", "043102"] + ["040202", "040502", "040802", "042002", "043102"], + ["2015-12-14", "2013-08-09", "2013-10-11", "2016-02-25"], + ["April-8-09"] ] for (let c of testCases) dateTimeExpandTest(c) } \ No newline at end of file From 524685a62abf2504faea0a89b53bc5c6daf0c397 Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Sat, 27 Aug 2022 17:57:36 +0800 Subject: [PATCH 11/17] test: add test cases (e.g.: MM.DD, YYYY) --- .../src/dev/workers/engine/dateTimeExpand.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index 842cf734..250559d3 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -138,7 +138,7 @@ export class DateTimeSynAnalyzer { this.rules = [ new SynAnalyzerRule(this.lex, [s.YEAR, s.MONTH, s.DATE, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/\\s](${r[1]})(?:[-/\\s](${r[2]}))?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s\\.,](${r[1]})(?:[-/\\s\\.](${r[2]}))?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // YYYY-MM-DD [HH[:mm[:ss[.ms]]]] new SynAnalyzerRule(this.lex, [s.YEAR, s.MONTH2, s.DATE2, s.HOUR, s.MIN, s.SEC, s.MS], @@ -146,7 +146,7 @@ export class DateTimeSynAnalyzer { ), // YYYYMMDD [HH[:mm[:ss[.ms]]]] new SynAnalyzerRule(this.lex, [s.YEAR2, s.MONTH, s.DATE, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/\\s](${r[1]})(?:[-/\\s](${r[2]}))?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s\\.,](${r[1]})(?:[-/\\s\\.](${r[2]}))?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // YY-MM-DD [HH[:mm[:ss[.ms]]]] new SynAnalyzerRule(this.lex, [s.YEAR2, s.MONTH2, s.DATE2, s.HOUR, s.MIN, s.SEC, s.MS], @@ -154,7 +154,7 @@ export class DateTimeSynAnalyzer { ), // YYMMDD [HH[:mm[:ss[.ms]]]] new SynAnalyzerRule(this.lex, [s.MONTH, s.DATE, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/\\s](${r[1]})(?:[-/\\s](${r[2]}))?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s\\.](${r[1]})(?:[-/\\s,](${r[2]}))?(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // MM-DD-YYYY new SynAnalyzerRule(this.lex, [s.MONTH2, s.DATE2, s.YEAR, s.HOUR, s.MIN, s.SEC, s.MS], @@ -162,7 +162,7 @@ export class DateTimeSynAnalyzer { ), // MMDDYYYY new SynAnalyzerRule(this.lex, [s.MONTH, s.DATE, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], - (r: string[]) => `^(${r[0]})[-/\\s](${r[1]})[-/\\s](${r[2]})(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` + (r: string[]) => `^(${r[0]})[-/\\s\\.](${r[1]})[-/\\s\\,](${r[2]})(?:[Tt\\s]+(${r[3]}))?(${r[4]})?(${r[5]})?(${r[6]})?$` ), // MM-DD-YY new SynAnalyzerRule(this.lex, [s.MONTH2, s.DATE2, s.YEAR2, s.HOUR, s.MIN, s.SEC, s.MS], @@ -353,7 +353,9 @@ export function doTest() { ["1998/4/9"], ["040202", "040502", "040802", "042002", "043102"], ["2015-12-14", "2013-08-09", "2013-10-11", "2016-02-25"], - ["April-8-09"] + ["April-8-09"], + ["04.29,2004", "12.29, 2008"], + ["2008,08.24"] ] for (let c of testCases) dateTimeExpandTest(c) } \ No newline at end of file From d2a2b302e408cb113a60b7ddbe1af85995d4ca92 Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Wed, 31 Aug 2022 17:26:46 +0800 Subject: [PATCH 12/17] - refactor: merge expandDateTime into [extend] button --- packages/rath-client/public/locales/en-US.json | 5 +---- packages/rath-client/public/locales/zh-CN.json | 5 +---- packages/rath-client/src/pages/dataSource/index.tsx | 13 +++---------- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index f0bc675f..23833ac4 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -166,10 +166,7 @@ "title": "Download Data" }, "extend": { - "title": "Extend Features" - }, - "expandDateTime": { - "title": "Expand DateTime Fields" + "title": "Extend Information" } }, "meta": { diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index 18b10714..353380ed 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -166,10 +166,7 @@ "title": "下载数据" }, "extend": { - "title": "扩展维度" - }, - "expandDateTime": { - "title": "扩展时间字段" + "title": "扩展信息" } }, "meta": { diff --git a/packages/rath-client/src/pages/dataSource/index.tsx b/packages/rath-client/src/pages/dataSource/index.tsx index c86da7cc..3478a629 100644 --- a/packages/rath-client/src/pages/dataSource/index.tsx +++ b/packages/rath-client/src/pages/dataSource/index.tsx @@ -247,20 +247,13 @@ const DataSourceBoard: React.FC = (props) => { /> { - dataSourceStore.extendData(); - }} - /> - { + // dataSourceStore.extendData(); + // TODO: 更多的扩展方式 dataSourceStore.expandDateTime(); }} - iconProps={{ iconName: 'CalendarMirrored' }} /> From 580674644bcdd8ad5b417c5fc75a8f7062e4fe88 Mon Sep 17 00:00:00 2001 From: "Asm.Def" Date: Wed, 31 Aug 2022 17:31:03 +0800 Subject: [PATCH 13/17] feat: mark some useless datetime fields as 'disable=true' after dateTimeExpand --- .../src/dev/workers/dateTimeExpand.worker.js | 2 +- .../src/dev/workers/engine/dateTimeExpand.ts | 39 ++++++++++++++----- .../rath-client/src/workers/engine/cleaner.ts | 3 +- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/packages/rath-client/src/dev/workers/dateTimeExpand.worker.js b/packages/rath-client/src/dev/workers/dateTimeExpand.worker.js index 165575d7..29093b53 100644 --- a/packages/rath-client/src/dev/workers/dateTimeExpand.worker.js +++ b/packages/rath-client/src/dev/workers/dateTimeExpand.worker.js @@ -13,7 +13,7 @@ const expandService = e => { } catch (error) { self.postMessage({ success: false, - message: error.toString() + message: `[expandService]${error}\n${error.stack}` }) } } diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index 250559d3..9a1ca04c 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -1,4 +1,4 @@ -import { IRow } from "visual-insights"; +import { IRow, ISemanticType } from "visual-insights"; import { IMuteFieldBase } from "rath-client/src/interfaces"; import { LexAnalyzer, LexAnalyzerItem, SynAnalyzerRule } from "./synAnalyzer" import { checkExpandEnv } from "./checkExpandEnv"; @@ -286,6 +286,18 @@ export function dateTimeExpand(props: { dataSource: IRow[]; fields: IMuteFieldBa let extFields: IMuteFieldBase[] = [] let fieldIds = new Set(fields.map(f => (f.extInfo && f.extInfo?.extOpt === "dateTimeExpand") ? f.extInfo?.extFrom[0] : '')) + let keySemanticType: { [k: string]: ISemanticType } = { + utime: 'temporal', + $y: 'ordinal', + $M: 'ordinal', + $D: 'ordinal', + $W: 'ordinal', + $H: 'ordinal', + $m: 'ordinal', + $s: 'ordinal', + $ms:'ordinal', + $L: 'nominal' + } for (let i = 0;i < fields.length; ++i) { const field = fields[i] extFields.push(field) @@ -293,25 +305,32 @@ export function dateTimeExpand(props: { dataSource: IRow[]; fields: IMuteFieldBa if (field.semanticType === 'temporal' && !fieldIds.has(field.fid)) { let dateTime = dataSource.map(item => item[field.fid]) let moment: DateTimeInfoArray = parseDateTimeArray(dateTime) - Object.keys(moment).forEach(key => { + if (moment.utime !== undefined) { + field.disable = true + if (moment.utime?.map(t => typeof(t) === 'number').reduce((a, b) => a && b) === false) { + throw new Error("[dateTimeExpand] Some 'utime' not number") + } + } + for (let key of Object.keys(moment)) if (moment[key as InfoArrayType] !== undefined) { + let momentInfo = moment[key as InfoArrayType] as any[] let extField: IMuteFieldBase = { fid: `${field.fid}_${key}`, + disable: (new Set(momentInfo.filter((info, index, array) => !index || info !== array[index-1]))).size <= 1, name: `${field.name}.${dateTimeDict.get(key)}`, analyticType: 'dimension', - semanticType: 'ordinal', + semanticType: keySemanticType[key], geoRole: 'none', extInfo: { extFrom: [field.fid], extOpt: 'dateTimeExpand', - extInfo: `${key}.value` + extInfo: `${key}` } } extFields.push(extField) - let infoArray = moment[key as InfoArrayType] as any[] for (let i = 0; i < dataSource.length; ++i) { - dataSource[i][extField.fid] = infoArray[i] + dataSource[i][extField.fid] = momentInfo[i] } - }) + } } } return { @@ -335,9 +354,9 @@ export function dateTimeExpandTest(testCase: Array) { } } -if (typeof window === 'object' && checkExpandEnv().toLowerCase() === 'debug') { - (window as any).parseDateTimeArray = parseDateTimeArray; - (window as any).dateTimeExpandTest = dateTimeExpandTest +if (checkExpandEnv().toLowerCase() === 'debug') { + (global as any).parseDateTimeArray = parseDateTimeArray; + (global as any).dateTimeExpandTest = dateTimeExpandTest } export function doTest() { diff --git a/packages/rath-client/src/workers/engine/cleaner.ts b/packages/rath-client/src/workers/engine/cleaner.ts index 689bfe43..d4852763 100644 --- a/packages/rath-client/src/workers/engine/cleaner.ts +++ b/packages/rath-client/src/workers/engine/cleaner.ts @@ -14,7 +14,8 @@ function transformDataTypes (dataSource: IRow[], fields: IRawField[]): IRow[] { // } record[field.fid] = row[field.fid] if (field.analyticType === 'dimension') { - if (field.semanticType === 'temporal' || field.semanticType === 'nominal') { + if ((field.semanticType === 'temporal' && field.extInfo?.extOpt !== "dateTimeExpand") || + field.semanticType === 'nominal') { record[field.fid] = String(row[field.fid]) } } From c14df349940f7c5f9efe1626d6b30f6bdcc0c5ab Mon Sep 17 00:00:00 2001 From: observedobserver <270001151@qq.com> Date: Fri, 2 Sep 2022 12:35:20 +0800 Subject: [PATCH 14/17] fix: $refresh bug --- .../rath-client/src/dev/workers/engine/dateTimeExpand.ts | 6 +++--- packages/rath-client/src/store/dataSourceStore.ts | 1 + packages/rath-client/src/workers/engine/metaInfer.ts | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts index 9a1ca04c..d7457470 100644 --- a/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts +++ b/packages/rath-client/src/dev/workers/engine/dateTimeExpand.ts @@ -202,7 +202,7 @@ export function parseReg(value: string, reg_id: number): DateTimeInfo { return {} as DateTimeInfo } -function Date2Info(date: Date): DateTimeInfo { +function date2Info(date: Date): DateTimeInfo { return { utime: date.getTime(), $y: date.getUTCFullYear(), @@ -223,12 +223,12 @@ function parseDateTime(dateTime: string, reg_id?: number): DateTimeInfo { if (dateTime === null) return UnknownDateTimeInfo; if (dateTime === undefined) { let date = new Date() - return Date2Info(date) + return date2Info(date) } // if (!/Z$/i.test(dateTime)) { // } // Polyfill - return Date2Info(new Date(dateTime)) + return date2Info(new Date(dateTime)) } catch(error) { console.warn(error) diff --git a/packages/rath-client/src/store/dataSourceStore.ts b/packages/rath-client/src/store/dataSourceStore.ts index f223af42..4478132d 100644 --- a/packages/rath-client/src/store/dataSourceStore.ts +++ b/packages/rath-client/src/store/dataSourceStore.ts @@ -466,6 +466,7 @@ export class DataSourceStore { this.mutFields = res.fields }) } catch (error) { + console.error(error) notify({ title: 'Expand DateTime API Error', type: 'error', diff --git a/packages/rath-client/src/workers/engine/metaInfer.ts b/packages/rath-client/src/workers/engine/metaInfer.ts index e9a93137..b16b732b 100644 --- a/packages/rath-client/src/workers/engine/metaInfer.ts +++ b/packages/rath-client/src/workers/engine/metaInfer.ts @@ -1,8 +1,8 @@ import { IAnalyticType, IRow, ISemanticType } from "visual-insights"; import { IGeoRole, IMuteFieldBase, IRawField } from "../../interfaces"; -import { inferAnalyticType, inferAnalyticTypeFromSemanticType, inferSemanticType } from "../../utils"; +import { inferAnalyticTypeFromSemanticType, inferSemanticType } from "../../utils"; -export function emptyCount (dataSource: IRow[], colKey: string): number { +function emptyCount (dataSource: IRow[], colKey: string): number { // const counter: Map = new Map(); let counter = 0; for (let i = 0; i < dataSource.length; i++) { From 7f1d1f165d84b2f43709d4765ac41f344ea18949 Mon Sep 17 00:00:00 2001 From: observedobserver <270001151@qq.com> Date: Fri, 2 Sep 2022 22:54:47 +0800 Subject: [PATCH 15/17] =?UTF-8?q?fix:=20=E9=9D=9E=E5=AF=86=E9=93=BA?= =?UTF-8?q?=E5=80=BC=E6=8A=98=E7=8E=B0=E5=9B=BE=E4=B8=8D=E8=BF=9E=E7=BB=AD?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/queries/distribution/bot.ts | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/rath-client/src/queries/distribution/bot.ts b/packages/rath-client/src/queries/distribution/bot.ts index bcc64a7d..f5c17a68 100644 --- a/packages/rath-client/src/queries/distribution/bot.ts +++ b/packages/rath-client/src/queries/distribution/bot.ts @@ -1,4 +1,5 @@ -import { ISemanticType } from "@kanaries/loa"; +import { IRow, ISemanticType } from "@kanaries/loa"; +import { groupBy } from "visual-insights/build/esm/statistics"; import { IFieldMeta, IVegaSubset } from "../../interfaces"; interface IFieldEncode { field?: string; @@ -16,12 +17,13 @@ function isSetEqual (a1: any[], a2: any[]) { } return true; } -export function autoMark (fields: IFieldMeta[], statFields: IFieldMeta[]= [], originFields: IFieldMeta[] = [], statEncodes: IFieldEncode[] = []) { +export function autoMark (fields: IFieldMeta[], statFields: IFieldMeta[]= [], originFields: IFieldMeta[] = [], statEncodes: IFieldEncode[] = [], dataSource?: IRow[]) { // const orderFields = [...fields]; // const orderStatFields = [...statFields]; // orderFields.sort((a, b) => b.features.entropy - a.features.entropy); // orderStatFields.sort((a, b) => b.features.entropy - a.features.entropy); - const semanticFields = [...statFields, ...originFields].sort((a, b) => b.features.entropy - a.features.entropy).slice(0, 2); + const allFields = [...statFields, ...originFields].sort((a, b) => b.features.entropy - a.features.entropy); + const semanticFields = allFields.slice(0, 2); const semantics = semanticFields.map(f => f.semanticType) // if (fields.length === 1) { // return 'bar' @@ -52,7 +54,17 @@ export function autoMark (fields: IFieldMeta[], statFields: IFieldMeta[]= [], or } return 'bar' } else if (isSetEqual(semantics, ['ordinal', 'quantitative'])) { - return 'bar' + if (dataSource) { + const dims = allFields.filter(f => f.analyticType === 'dimension'); + const dsize = dims.reduce((c, t) => c * t.features.unique, 1); + const groups = groupBy(dataSource, allFields.filter(f => f.analyticType === 'dimension').map(f => f.fid)); + if (groups.size < dsize) { + return 'bar' + } + return 'line' + } + + return 'line' } else if (isSetEqual(semantics, ['nominal', 'ordinal'])) { return 'point' } else if (isSetEqual(semantics, ['nominal', 'temporal'])) { @@ -236,6 +248,13 @@ export function encode(props: EncodeProps) { } export function humanHabbit (encoding: IVegaSubset['encoding']) { + if (encoding.x && encoding.x.type !== 'ordinal') { + if (encoding.y && encoding.y.type === 'ordinal') { + const t = encoding.x; + encoding.x = encoding.y; + encoding.y = t; + } + } if (encoding.x && encoding.x.type !== 'temporal') { if (encoding.y && encoding.y.type === 'temporal') { const t = encoding.x; From 4d2f4e6ce2be7e3ed53cb4a46eb50df243ab1639 Mon Sep 17 00:00:00 2001 From: observedobserver <270001151@qq.com> Date: Fri, 2 Sep 2022 22:55:34 +0800 Subject: [PATCH 16/17] =?UTF-8?q?fix:=20=E5=85=A8=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E5=8F=AF=E4=BB=A5=E8=87=AA=E5=8A=A8=E6=84=8F?= =?UTF-8?q?viz=E6=8E=A8=E8=8D=90=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/pages/megaAutomation/preference.tsx | 21 +++++++- .../rath-client/src/queries/labdistVis.ts | 8 +-- .../rath-client/src/store/exploreStore.ts | 49 ++++++++++++++----- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/packages/rath-client/src/pages/megaAutomation/preference.tsx b/packages/rath-client/src/pages/megaAutomation/preference.tsx index 2d4f9a25..faca94c6 100644 --- a/packages/rath-client/src/pages/megaAutomation/preference.tsx +++ b/packages/rath-client/src/pages/megaAutomation/preference.tsx @@ -11,6 +11,8 @@ import { Toggle, SpinButton, Position, + ChoiceGroup, + IChoiceGroupOption, } from 'office-ui-fabric-react'; import intl from 'react-intl-universal'; import { useGlobalStore } from '../../store'; @@ -19,7 +21,7 @@ import { IResizeMode } from '../../interfaces'; const PreferencePanel: React.FC = () => { const { exploreStore } = useGlobalStore(); - const { visualConfig, showPreferencePannel, nlgThreshold } = exploreStore; + const { visualConfig, showPreferencePannel, nlgThreshold, vizMode } = exploreStore; const { nlg } = visualConfig; @@ -51,6 +53,13 @@ const PreferencePanel: React.FC = () => { ); + const vizModeOptions = useMemo(() => { + return [ + { text: intl.get('discovery.main.vizsys.lite'), key: 'lite' }, + { text: intl.get('discovery.main.vizsys.strict'), key: 'strict' } + ] + }, []) + return ( { closeButtonAriaLabel="Close" onRenderFooterContent={onRenderFooterContent} > + + { + op && exploreStore.setVizMode(op.key as 'lite' | 'strict') + }} + selectedKey={vizMode} + options={vizModeOptions} + /> + f.analyticType === 'measure'); const dimensions = fields.filter(f => f.analyticType === 'dimension'); @@ -144,7 +145,7 @@ export function labDistVis(props: BaseVisProps): IVegaSubset { dimensions[i].features.entropy = totalEntLoss; } const { statFields, distFields, statEncodes } = autoStat(fields); - let markType = autoMark(fields, statFields, distFields, statEncodes) + let markType = autoMark(fields, statFields, distFields, statEncodes, dataSource) const channelEncoder = new VizEncoder(markType); // if (filters && filters.length > 0) { // usedChannels.add('color') @@ -202,7 +203,8 @@ export function labDistVis(props: BaseVisProps): IVegaSubset { applySizeConfig2DistViz(basicSpec, { mode: resizeMode, width, - height + height, + stepSize }) if (interactive) { applyInteractiveParams2DistViz(basicSpec); diff --git a/packages/rath-client/src/store/exploreStore.ts b/packages/rath-client/src/store/exploreStore.ts index 5ae57210..b9210f18 100644 --- a/packages/rath-client/src/store/exploreStore.ts +++ b/packages/rath-client/src/store/exploreStore.ts @@ -4,6 +4,7 @@ import { Specification, IInsightSpace, ISpec } from 'visual-insights'; import { STORAGE_FILE_SUFFIX } from '../constants'; import { IResizeMode, IRow, ITaskTestMode, IVegaSubset, PreferencePanelConfig } from '../interfaces'; import { distVis } from '../queries/distVis'; +import { labDistVis } from '../queries/labdistVis'; import { rathEngineService } from '../service'; import { isSetEqual } from '../utils'; import { RathStorageDump } from '../utils/storage'; @@ -44,6 +45,7 @@ export class ExploreStore { public mainViewPattern: IPattern | null = null; public orderBy: string = EXPLORE_VIEW_ORDER.DEFAULT; public nlgThreshold: number = 0.2; + public vizMode: 'lite' | 'strict' = 'lite'; public globalConstraints: { dimensions: Array; measures: Array @@ -123,6 +125,10 @@ export class ExploreStore { public setNlgThreshold (num: number) { this.nlgThreshold = num; } + public setVizMode (mode: 'lite' | 'strict') { + this.vizMode = mode; + this.refreshMainViewSpec(); + } public setVisualConig (updater: (config: PreferencePanelConfig) => void) { runInAction(() => { updater(this.visualConfig) @@ -196,22 +202,39 @@ export class ExploreStore { } return this.mainViewPattern; } - public createMainViewSpec (pattern: IPattern, mode: 'lite' | 'strict') { - const { visualConfig } = this; - this.mainViewSpec = distVis({ - resizeMode: visualConfig.resize, - pattern, - width: visualConfig.resizeConfig.width, - height: visualConfig.resizeConfig.height, - interactive: visualConfig.zoom, - stepSize: 32 - }) + public createMainViewSpec (pattern: IPattern) { + const { visualConfig, vizMode } = this; + if (vizMode === 'lite') { + this.mainViewSpec = distVis({ + resizeMode: visualConfig.resize, + pattern, + width: visualConfig.resizeConfig.width, + height: visualConfig.resizeConfig.height, + interactive: visualConfig.zoom, + stepSize: 32 + }) + } else if (vizMode === 'strict') { + this.mainViewSpec = labDistVis({ + resizeMode: visualConfig.resize, + pattern, + width: visualConfig.resizeConfig.width, + height: visualConfig.resizeConfig.height, + interactive: visualConfig.zoom, + stepSize: 32, + dataSource: this.dataSource + }) + } + } + public refreshMainViewSpec () { + if (this.mainViewPattern) { + this.createMainViewSpec(this.mainViewPattern) + } } public addField2MainViewPattern (fid: string) { const targetField = this.fieldMetas.find(f => f.fid === fid); if (targetField && this.mainViewPattern) { this.mainViewPattern.fields.push(targetField); - this.createMainViewSpec(this.mainViewPattern, 'lite') + this.createMainViewSpec(this.mainViewPattern) } } public removeFieldInViewPattern (fid: string) { @@ -219,7 +242,7 @@ export class ExploreStore { const targetFieldIndex = this.mainViewPattern.fields.findIndex(f => f.fid === fid); if (targetFieldIndex > -1) { this.mainViewPattern.fields.splice(targetFieldIndex, 1) - this.createMainViewSpec(this.mainViewPattern, 'lite') + this.createMainViewSpec(this.mainViewPattern) } } } @@ -236,7 +259,7 @@ export class ExploreStore { if (this.insightSpaces && this.insightSpaces.length > index) { const iSpace = this.insightSpaces[index]; const patt = this.createMainViewPattern(iSpace); - this.createMainViewSpec(patt, 'lite'); + this.createMainViewSpec(patt); this.pageIndex = index; this.details = [] this.showAsso = false; From 57386773309cb8b66154de09997664c1703d5d9e Mon Sep 17 00:00:00 2001 From: observedobserver <270001151@qq.com> Date: Fri, 2 Sep 2022 22:55:51 +0800 Subject: [PATCH 17/17] =?UTF-8?q?fix:=20asociation=20=E4=B8=A2=E5=A4=B1bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/pages/megaAutomation/assoPanel.tsx | 20 +++++++++++++++++++ .../src/pages/megaAutomation/index.tsx | 2 ++ 2 files changed, 22 insertions(+) create mode 100644 packages/rath-client/src/pages/megaAutomation/assoPanel.tsx diff --git a/packages/rath-client/src/pages/megaAutomation/assoPanel.tsx b/packages/rath-client/src/pages/megaAutomation/assoPanel.tsx new file mode 100644 index 00000000..163c4965 --- /dev/null +++ b/packages/rath-client/src/pages/megaAutomation/assoPanel.tsx @@ -0,0 +1,20 @@ +import React from 'react'; +import { observer } from 'mobx-react-lite'; +import { Panel, PanelType } from 'office-ui-fabric-react'; +import { useGlobalStore } from '../../store'; +import Association from './association'; + +const AssoPanel: React.FC = () => { + const { exploreStore } = useGlobalStore(); + return
+ { + exploreStore.setShowAsso(false); + }}> + + +
+} + +export default observer(AssoPanel); diff --git a/packages/rath-client/src/pages/megaAutomation/index.tsx b/packages/rath-client/src/pages/megaAutomation/index.tsx index 10266f4a..dbbaac99 100644 --- a/packages/rath-client/src/pages/megaAutomation/index.tsx +++ b/packages/rath-client/src/pages/megaAutomation/index.tsx @@ -19,6 +19,7 @@ import { LoadingLayer } from '../semiAutomation/components'; import ComputationProgress from './computationProgress'; import ReactVega from '../../components/react-vega'; import Constraints from './vizOperation/constraints'; +import AssoPanel from './assoPanel'; const MainHeader = styled.div` font-size: 1.5em; @@ -86,6 +87,7 @@ const LTSPage: React.FC = () => { + {/* { exploreStore.setShowSubinsights(false) }} /> */}