feat: update importer version (#502)

* mod: update importer version * mod: update import logic mod: code review * mod: add withhaeder & remove vidprefix * mod: code review
vesoft-inc · Mar 16, 2023 · 7fd290a · 7fd290a
1 parent 9f07a3d
commit 7fd290a
Show file tree

Hide file tree

Showing 23 changed files with 718 additions and 786 deletions.
diff --git a/app/components/CSVPreviewLink/index.tsx b/app/components/CSVPreviewLink/index.tsx
@@ -21,7 +21,6 @@ const CSVPreviewLink = (props: IProps) => {
   const [data, setData] = useState<any[]>([]);
   const { intl } = useI18n();
   const { readString } = usePapaParse();
-
   useEffect(() => {
     if(!file) return;
     const { delimiter, sample } = file;

diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json
@@ -190,8 +190,7 @@
     "importFailed": "Failed",
     "importRunning": "Running",
     "importPending": "Pending",
-    "notImported": "{total} lines not imported",
-    "readFailed": "{total} lines read failed",
+    "notImported": "{total} records not imported",
     "selectFile": "Select file",
     "addTag": "Add Tag",
     "addEdge": "Add Edge Type",
@@ -218,18 +217,15 @@
     "srcVidColumn": "Source VID column",
     "dstVidColumn": "Destination VID column",
     "vidFunction": "VID function",
-    "vidPrefix": "VID prefix",
     "concurrencyTip": "Number of NebulaGraph client concurrency.",
     "batchSizeTip": "The number of statements inserting data in a batch.",
     "retryTip": "Retry times of nGQL statement execution failures.",
-    "channelBufferSizeTip": "Cache queue size per NebulaGraph client.",
     "vidFunctionTip": "Function to generate VID. Currently only hash functions are supported.",
     "vidPrefixTip": "prefix added to the original vid.",
     "selectCsvColumn": "Select CSV Index",
     "graphAddress": "Graph service address",
     "concurrency": "Concurrency",
     "retry": "Retry",
-    "channelBufferSize": "Channel Buffer Size",
     "graphAddressTip": "The following Graph host will be used for data import",
     "currentHost": "Current connected host",
     "expandMoreConfig": "Expand more configurations",

diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json
@@ -190,8 +190,7 @@
     "importFailed": "导入失败",
     "importRunning": "导入中",
     "importPending": "等待导入",
-    "notImported": "{total}行未导入",
-    "readFailed": "{total}行读取失败",
+    "notImported": "{total}条记录未导入",
     "selectFile": "选择绑定文件",
     "addTag": "添加 Tag",
     "addEdge": "添加 Edge Type",
@@ -218,18 +217,15 @@
     "srcVidColumn": "起点 VID 列",
     "dstVidColumn": "终点 VID 列",
     "vidFunction": "VID 函数",
-    "vidPrefix": "VID 前缀",
     "concurrencyTip": "NebulaGraph 客户端并发数",
     "batchSizeTip": "单批次插入数据的语句数量",
     "retryTip": "nGQL 语句执行失败的重试次数",
-    "channelBufferSizeTip": "每个 NebulaGraph 客户端的缓存队列大小",
     "vidFunctionTip": "生成 VID 的函数。目前只支持 hash 函数",
     "vidPrefixTip": "给原始 VID 添加的前缀",
     "selectCsvColumn": "选择 CSV 列",
     "graphAddress": "Graph 服务地址",
     "concurrency": "并发数",
     "retry": "重试次数",
-    "channelBufferSize": "缓存队列大小",
     "graphAddressTip": "Graph 服务的地址和端口。将使用以下 Graph 节点进行数据导入",
     "currentHost": "当前登录的 Graph 节点",
     "expandMoreConfig": "展开更多配置",

diff --git a/app/interfaces/import.ts b/app/interfaces/import.ts
@@ -10,13 +10,16 @@ export enum ITaskStatus {
 }
 
 export interface ITaskStats {
-  totalBatches: number;
+  processedBytes: number;
   totalBytes: number;
-  totalImportedBytes: number;
+  failedRecords: number;
+  totalRecords: number;
+  failedRequest: number;
+  totalRequest: number;
   totalLatency: number;
-  totalReqTime: number;
-  numFailed: number;
-  numReadFailed: number;
+  totalRespTime: number;
+  failedProcessed: number; // The number of nodes and edges that have failed to be processed.
+  totalProcessed: number; // The number of nodes and edges that have been processed.
 }
 export interface ITaskItem {
   id: number;
@@ -52,7 +55,6 @@ export interface IBasicConfig {
   batchSize?: string;
   concurrency?: string;
   retry?: string;
-  channelBufferSize?: string;
 }
 
 export interface ILogDimension {

diff --git a/app/pages/Import/FileList/PreviewFileModal/index.tsx b/app/pages/Import/FileList/PreviewFileModal/index.tsx
@@ -1,5 +1,5 @@
 import { useI18n } from '@vesoft-inc/i18n';
-import { Button, Input, Modal, Table, Form, Row, Col, message } from 'antd';
+import { Button, Input, Modal, Table, Form, Row, Col, message, Checkbox } from 'antd';
 import { v4 as uuidv4 } from 'uuid';
 import React, { PropsWithChildren, useCallback, useEffect } from 'react';
 import { usePapaParse } from 'react-papaparse';
@@ -92,11 +92,11 @@ const PreviewFileModal = (props: IProps) => {
             delimiter,
           }}>
             <Row className={styles.configOperation}>
-              {/* <Col span={3}>
+              <Col span={3}>
                 <Form.Item name="withHeader" valuePropName="checked">
                   <Checkbox onChange={updateHeader}>{intl.get('import.hasHeader')}</Checkbox>
                 </Form.Item>
-              </Col> */}
+              </Col>
               <Col span={8}>
                 <Form.Item label={intl.get('import.delimiter')} name="delimiter" required={true}>
                   <Input placeholder="," onChange={handlePreview} />

diff --git a/app/pages/Import/FileList/UploadConfigModal/index.tsx b/app/pages/Import/FileList/UploadConfigModal/index.tsx
@@ -1,6 +1,6 @@
 import Icon from '@app/components/Icon';
 import { useI18n } from '@vesoft-inc/i18n';
-import { Button, Input, Modal, Table, Popconfirm, Dropdown } from 'antd';
+import { Button, Input, Modal, Table, Popconfirm, Dropdown, message } from 'antd';
 import { v4 as uuidv4 } from 'uuid';
 import React, { useCallback, useEffect, useState } from 'react';
 import { usePapaParse } from 'react-papaparse';
@@ -10,6 +10,7 @@ import { useStore } from '@app/stores';
 import { observer, useLocalObservable } from 'mobx-react-lite';
 import { ExclamationCircleFilled } from '@ant-design/icons';
 import { observable } from 'mobx';
+import Checkbox, { CheckboxChangeEvent } from 'antd/lib/checkbox';
 import styles from './index.module.less';
 interface IProps {
   visible: boolean;
@@ -38,8 +39,8 @@ const UploadConfigModal = (props: IProps) => {
     data: [],
     activeItem: null,
     previewContent: [],
-    // checkAll: false,
-    // indeterminate: false,
+    checkAll: false,
+    indeterminate: false,
     loading: false,
     uploading: false,
     setState: (obj) => Object.assign(state, obj),
@@ -73,36 +74,36 @@ const UploadConfigModal = (props: IProps) => {
     });
   }, []);
 
-  // const onCheckAllChange = useCallback((e: CheckboxChangeEvent) => {
-  //   const { data, setState } = state;
-  //   const { checked } = e.target;
-  //   setState({
-  //     checkAll: checked,
-  //     indeterminate: false,
-  //     data: data.map(i => (i.withHeader = checked, i))
-  //   });
-  // }, []);
+  const onCheckAllChange = useCallback((e: CheckboxChangeEvent) => {
+    const { data, setState } = state;
+    const { checked } = e.target;
+    setState({
+      checkAll: checked,
+      indeterminate: false,
+      data: data.map(i => (i.withHeader = checked, i))
+    });
+  }, []);
 
-  // const updateItem = useCallback((e: CheckboxChangeEvent, item: StudioFile) => {
-  //   const { data, setState } = state;
-  //   const { checked } = e.target;
-  //   const nextData = data.map(i => (i === item && (i.withHeader = checked), i));
-  //   const checkedNum = data.reduce((n, file) => n + (file.withHeader & 1), 0);
-  //   setState({
-  //     checkAll: checkedNum === data.length,
-  //     indeterminate: !!checkedNum && checkedNum < data.length,
-  //     data: nextData
-  //   });
-  // }, []);
+  const updateItem = useCallback((e: CheckboxChangeEvent, item: StudioFile) => {
+    const { data, setState } = state;
+    const { checked } = e.target;
+    const nextData = data.map(i => (i === item && (i.withHeader = checked), i));
+    const checkedNum = data.reduce((n, file) => n + (file.withHeader & 1), 0);
+    setState({
+      checkAll: checkedNum === data.length,
+      indeterminate: !!checkedNum && checkedNum < data.length,
+      data: nextData
+    });
+  }, []);
   const deletePreviewFile = useCallback((e: React.MouseEvent, index: number) => {
     const { activeItem, data, setState, previewContent } = state;
     e.stopPropagation();
     const isActive = activeItem?.uid === data[index].uid;
     const newData = data.filter((_, i) => i !== index);
-    // const checkedNum = data.reduce((n, file) => n + (file.withHeader & 1), 0);
+    const checkedNum = data.reduce((n, file) => n + (file.withHeader & 1), 0);
     setState({
-      // checkAll: checkedNum === newData.length && newData.length > 0,
-      // indeterminate: !!checkedNum && checkedNum < newData.length,
+      checkAll: checkedNum === newData.length && newData.length > 0,
+      indeterminate: !!checkedNum && checkedNum < newData.length,
       data: newData,
       activeItem: isActive ? null : activeItem,
       previewContent: isActive ? [] : previewContent,
@@ -158,6 +159,7 @@ const UploadConfigModal = (props: IProps) => {
     const res = await uploadFile(data);
     if(res.code === 0) {
       onConfirm();
+      message.success(intl.get('import.uploadSuccessfully'));
     }
     setState({ uploading: false });
   }, []);
@@ -169,7 +171,7 @@ const UploadConfigModal = (props: IProps) => {
   if(!visible) {
     return null;
   }
-  const { uploading, data, activeItem, previewContent, loading, setState } = state;
+  const { uploading, data, activeItem, previewContent, loading, setState, checkAll, indeterminate } = state;
   const parseColumns = previewContent.length
     ? previewContent[0].map((header, index) => {
       const textIndex = index;
@@ -185,18 +187,18 @@ const UploadConfigModal = (props: IProps) => {
     {
       title: intl.get('import.fileName'),
       dataIndex: 'name',
-      // align: 'center' as const,
-      width: '60%'
+      align: 'center' as const,
+      width: '30%'
+    },
+    {
+      title: <>
+        <Checkbox checked={checkAll} indeterminate={indeterminate} onChange={onCheckAllChange} />
+        <span style={{ paddingLeft: '5px' }}>{intl.get('import.withHeader')}</span>
+      </>,
+      key: 'withHeader',
+      width: '30%',
+      render: record => <Checkbox checked={record.withHeader} onChange={e => updateItem(e, record)}>{intl.get('import.hasHeader')}</Checkbox>,
     },
-    // {
-    //   title: <>
-    //     <Checkbox checked={checkAll} indeterminate={indeterminate} onChange={onCheckAllChange} />
-    //     <span style={{ paddingLeft: '5px' }}>{intl.get('import.withHeader')}</span>
-    //   </>,
-    //   key: 'withHeader',
-    //   width: '30%',
-    //   render: record => <Checkbox checked={record.withHeader} onChange={e => updateItem(e, record)}>{intl.get('import.hasHeader')}</Checkbox>,
-    // },
     {
       title: <>
         <span>{intl.get('import.delimiter')}</span>

diff --git a/app/pages/Import/FileList/index.tsx b/app/pages/Import/FileList/index.tsx
@@ -26,13 +26,13 @@ const FileList = () => {
     {
       title: intl.get('import.fileName'),
       dataIndex: 'name',
-      width: '40%'
+      width: '50%'
+    },
+    {
+      title: intl.get('import.withHeader'),
+      dataIndex: 'withHeader',
+      render: value => value ? intl.get('import.hasHeader') : intl.get('import.noHeader'),
     },
-    // {
-    //   title: intl.get('import.withHeader'),
-    //   dataIndex: 'withHeader',
-    //   render: value => value ? intl.get('import.hasHeader') : intl.get('import.noHeader'),
-    // },
     {
       title: intl.get('import.delimiter'),
       dataIndex: 'delimiter',

diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx
@@ -1,4 +1,4 @@
-import { Collapse, Input, Select, Table, Tooltip } from 'antd';
+import { Collapse, Select, Table, Tooltip } from 'antd';
 import React, { useMemo } from 'react';
 import { observer } from 'mobx-react-lite';
 import cls from 'classnames';
@@ -53,10 +53,6 @@ const VIDSetting = observer((props: {
             <span className={styles.title}>{intl.get('import.vidFunction')}</span>
             <span>{intl.get('import.vidFunctionTip')}</span>
           </div>
-          <div>
-            <span className={styles.title}>{intl.get('import.vidPrefix')}</span>
-            <span>{intl.get('import.vidPrefixTip')}</span>
-          </div>
         </>} />
       </div>} key="default">
         {spaceVidType === 'INT64' && idFunction && <div className={styles.rowItem}>
@@ -75,10 +71,6 @@ const VIDSetting = observer((props: {
             <Option value="hash">Hash</Option>
           </Select>
         </div>}
-        {idPrefix && <div className={styles.rowItem}>
-          <span className={styles.label}>{intl.get('import.vidPrefix')}</span>
-          <Input className={styles.prefixInput} bordered={false} placeholder="Input prefix" value={data[idPrefix]} onChange={e => data.update({ [idPrefix]: e.target.value })} />
-        </div>}
       </Panel>
     </Collapse>
   </div>;
@@ -88,7 +80,6 @@ const idMap = {
   [ISchemaEnum.Tag]: [{
     idKey: 'vidIndex',
     idFunction: 'vidFunction',
-    idPrefix: 'vidPrefix',
     label: 'vidColumn'
   }],
   [ISchemaEnum.Edge]: [{

diff --git a/app/pages/Import/TaskCreate/index.tsx b/app/pages/Import/TaskCreate/index.tsx
@@ -206,18 +206,6 @@ const TaskCreate = (props: IProps) => {
       placeholder: DEFAULT_IMPORT_CONFIG.retry,
       description: intl.get('import.retryTip'),
     },
-    {
-      label: intl.get('import.channelBufferSize'),
-      key: 'channelBufferSize',
-      rules: [
-        {
-          pattern: POSITIVE_INTEGER_REGEX,
-          message: intl.get('formRules.numberRequired'),
-        },
-      ],
-      placeholder: DEFAULT_IMPORT_CONFIG.channelBufferSize,
-      description: intl.get('import.channelBufferSizeTip'),
-    },
   ], [currentLocale]);
   return (
     <div className={styles.importCreate}>

diff --git a/app/pages/Import/TaskList/TaskItem/index.tsx b/app/pages/Import/TaskList/TaskItem/index.tsx
@@ -57,16 +57,13 @@ const TaskItem = (props: IProps) => {
   const { intl } = useI18n();
   const [progressStatus, setStatus] = useState<'success' | 'active' | 'normal' | 'exception' | undefined>(undefined);
   const [extraMsg, setExtraMsg] = useState('');
-  const { totalImportedBytes, totalBytes, numFailed, numReadFailed } = stats || {};
+  const { processedBytes, totalBytes, failedProcessed } = stats || {};
   const time = useRef('');
   const timeoutId = useRef<number>(null);
   const addMsg = () => {
     const info: string[] = [];
-    if(numFailed > 0) {
-      info.push(intl.get('import.notImported', { total: numFailed }));
-    }
-    if(numReadFailed > 0) {
-      info.push(intl.get('import.readFailed', { total: numReadFailed }));
+    if(failedProcessed > 0) {
+      info.push(intl.get('import.notImported', { total: failedProcessed }));
     }
     info.length > 0 && setExtraMsg(info.join(', '));
   };
@@ -91,10 +88,10 @@ const TaskItem = (props: IProps) => {
   }, [status]);
   const refreshTime = () => {
     if(status === ITaskStatus.StatusProcessing) {
-      time.current = dayjs.duration(dayjs(Date.now()).diff(dayjs.unix(createTime))).format('HH:mm:ss');
+      time.current = dayjs.duration(dayjs(Date.now()).diff(dayjs(createTime))).format('HH:mm:ss');
       timeoutId.current = window.setTimeout(refreshTime, 1000);
     } else {
-      time.current = dayjs.duration(dayjs.unix(updateTime).diff(dayjs.unix(createTime))).format('HH:mm:ss');
+      time.current = dayjs.duration(dayjs(updateTime).diff(dayjs(createTime))).format('HH:mm:ss');
     }
   };
   return (
@@ -131,8 +128,8 @@ const TaskItem = (props: IProps) => {
               </span>}
             </span>
             <div className={styles.moreInfo}>
-              {totalImportedBytes > 0 && <span>
-                {status !== ITaskStatus.StatusFinished && `${getFileSize(totalImportedBytes)} / `}
+              {processedBytes > 0 && <span>
+                {status !== ITaskStatus.StatusFinished && `${getFileSize(processedBytes)} / `}
                 {getFileSize(totalBytes)}{' '}
               </span>}
               <span>{time.current}</span>
@@ -141,7 +138,7 @@ const TaskItem = (props: IProps) => {
           {stats && <Progress 
             format={percent => `${percent}%`}
             status={progressStatus} 
-            percent={status !== ITaskStatus.StatusFinished ? floor(totalImportedBytes / totalBytes * 100, 2) : 100} 
+            percent={status !== ITaskStatus.StatusFinished ? floor(processedBytes / totalBytes * 100, 2) : 100} 
             strokeColor={progressStatus && COLOR_MAP[progressStatus]} />}
         </div>
         <div className={styles.operations}>