From 8ae35e6d9e1b942e02192c54ecb68fa14f08cc45 Mon Sep 17 00:00:00 2001 From: zixi0825 Date: Sun, 10 Mar 2024 22:43:29 +0800 Subject: [PATCH] [Fix][Server] Fix some bug (#376) --- .../common/enums/ExecutionStatus.java | 85 +++++++++---------- .../common/param/GetTablesRequestParam.java | 2 +- .../connector/api/ConfigBuilder.java | 2 + .../connector/plugin/JdbcConfigBuilder.java | 5 ++ .../connector/plugin/JdbcConnector.java | 6 +- .../connector/plugin/MongodbConnector.java | 2 +- .../connector/plugin/MysqlConfigBuilder.java | 50 +++++++++++ .../connector/plugin/MysqlConnector.java | 1 + .../connector/plugin/OracleDialect.java | 2 +- .../connector/plugin/SqlServerDialect.java | 2 +- .../plugin/StarRocksConfigBuilder.java | 40 +++++++++ .../connector/plugin/StarRocksConnector.java | 70 +++++++++++++++ .../plugin/StarRocksConnectorFactory.java | 2 +- .../metric/api/MetricExecutionResult.java | 59 ++++++++++--- .../api/controller/MetricController.java | 33 +++++-- .../server/api/dto/vo/JobExecutionVO.java | 9 +- .../CatalogMetaDataFetchExecutorImpl.java | 2 +- .../operator/JobResultValidator.java | 6 +- .../server/enums/DqJobExecutionState.java | 13 ++- .../service/impl/DataSourceServiceImpl.java | 2 +- .../impl/ErrorDataStorageServiceImpl.java | 2 +- .../service/impl/JobExecutionServiceImpl.java | 32 +++---- .../resources/mapper/JobExecutionMapper.xml | 2 +- .../MetricModal/ExpectedValue/index.tsx | 5 +- datavines-ui/src/locale/en_US.ts | 5 +- datavines-ui/src/locale/zh_CN.ts | 11 +-- .../view/Main/HomeDetail/Dashboard/index.tsx | 15 +++- .../HomeDetail/JobExecutionLogs/index.tsx | 14 +-- .../Main/HomeDetail/Jobs/JobsInstance.tsx | 15 +++- scripts/sql/datavines-mysql.sql | 6 +- 30 files changed, 363 insertions(+), 137 deletions(-) create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConfigBuilder.java diff --git a/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java b/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java index 20c8380b2..97912a388 100644 --- a/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java +++ b/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java @@ -19,6 +19,7 @@ import java.util.HashMap; import com.baomidou.mybatisplus.annotation.EnumValue; +import lombok.Getter; /** * running status for workflow and task nodes @@ -41,27 +42,32 @@ public enum ExecutionStatus { * 10 waiting thread * 11 waiting depend node complete */ - SUBMITTED_SUCCESS(0, "submitted"), - RUNNING_EXECUTION(1, "running"), - READY_PAUSE(2, "ready pause"), - PAUSE(3, "pause"), - READY_STOP(4, "ready stop"), - STOP(5, "stop"), - FAILURE(6, "failure"), - SUCCESS(7, "success"), - NEED_FAULT_TOLERANCE(8, "need fault tolerance"), - KILL(9, "kill"), - WAITING_THREAD(10, "waiting thread"), - WAITING_DEPEND(11, "waiting depend node complete"); - - ExecutionStatus(int code, String description){ + SUBMITTED_SUCCESS(0, "submitted", "已提交"), + RUNNING_EXECUTION(1, "running", "执行中"), + READY_PAUSE(2, "ready pause", "准备暂停"), + PAUSE(3, "pause", "暂停"), + READY_STOP(4, "ready stop", "准备停止"), + STOP(5, "stop", "停止"), + FAILURE(6, "failure", "失败"), + SUCCESS(7, "success", "成功"), + NEED_FAULT_TOLERANCE(8, "need fault tolerance","需要容错"), + KILL(9, "kill", "强制终止"), + WAITING_THREAD(10, "waiting thread", "等待线程"), + WAITING_DEPEND(11, "waiting depend node complete",""); + + ExecutionStatus(int code, String description,String zhDescription){ this.code = code; this.description = description; + this.zhDescription = zhDescription; } + @Getter @EnumValue - int code; - String description; + private final int code; + @Getter + private final String description; + @Getter + private final String zhDescription; private static final HashMap EXECUTION_STATUS_MAP = new HashMap<>(); @@ -71,37 +77,37 @@ public enum ExecutionStatus { } } - /** - * status is success - * @return status - */ - public boolean typeIsSuccess(){ + /** + * status is success + * @return status + */ + public boolean typeIsSuccess(){ return this == SUCCESS; } - /** - * status is failure - * @return status - */ - public boolean typeIsFailure(){ + /** + * status is failure + * @return status + */ + public boolean typeIsFailure(){ return this == FAILURE || this == NEED_FAULT_TOLERANCE || this == KILL; } - /** - * status is finished - * @return status - */ - public boolean typeIsFinished(){ + /** + * status is finished + * @return status + */ + public boolean typeIsFinished(){ - return typeIsSuccess() || typeIsFailure() || typeIsCancel() || typeIsPause() + return typeIsSuccess() || typeIsFailure() || typeIsCancel() || typeIsPause() || typeIsStop(); - } + } /** * status is waiting thread * @return status */ - public boolean typeIsWaitingThread(){ + public boolean typeIsWaitingThread(){ return this == WAITING_THREAD; } @@ -109,7 +115,7 @@ public boolean typeIsWaitingThread(){ * status is pause * @return status */ - public boolean typeIsPause(){ + public boolean typeIsPause(){ return this == PAUSE; } /** @@ -124,7 +130,7 @@ public boolean typeIsStop(){ * status is running * @return status */ - public boolean typeIsRunning(){ + public boolean typeIsRunning(){ return this == RUNNING_EXECUTION || this == WAITING_DEPEND; } @@ -147,11 +153,4 @@ public boolean canPause(){ return this == SUBMITTED_SUCCESS || this == READY_PAUSE; } - public int getCode() { - return code; - } - - public String getDescription() { - return description; - } } diff --git a/datavines-common/src/main/java/io/datavines/common/param/GetTablesRequestParam.java b/datavines-common/src/main/java/io/datavines/common/param/GetTablesRequestParam.java index a812d8103..1a73cea9f 100644 --- a/datavines-common/src/main/java/io/datavines/common/param/GetTablesRequestParam.java +++ b/datavines-common/src/main/java/io/datavines/common/param/GetTablesRequestParam.java @@ -22,5 +22,5 @@ @EqualsAndHashCode(callSuper = true) @Data public class GetTablesRequestParam extends ConnectorRequestParam { - private String dataBase; + private String database; } diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConfigBuilder.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConfigBuilder.java index 70f400e58..d27129668 100644 --- a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConfigBuilder.java +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConfigBuilder.java @@ -19,4 +19,6 @@ public interface ConfigBuilder { String build(boolean isEn); + + String buildErrorDataStorage(boolean isEn); } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConfigBuilder.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConfigBuilder.java index f2f3bf0fe..496cf9eab 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConfigBuilder.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConfigBuilder.java @@ -66,6 +66,11 @@ public String build(boolean isEn) { return result; } + @Override + public String buildErrorDataStorage(boolean isEn) { + return this.build(isEn); + } + protected InputParam getHostInput(boolean isEn) { return getInputParam("host", isEn ? "host":"地址", diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnector.java index b0dce3e8a..e4ded4489 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnector.java @@ -60,7 +60,7 @@ public JdbcConnector(DataSourceClient dataSourceClient) { this.dataSourceClient = dataSourceClient; } - private Connection getConnection(String dataSourceParam, JdbcConnectionInfo jdbcConnectionInfo) throws SQLException { + protected Connection getConnection(String dataSourceParam, JdbcConnectionInfo jdbcConnectionInfo) throws SQLException { return dataSourceClient.getConnection(JdbcDataSourceInfoManager.getDatasourceInfo(dataSourceParam, getDatasourceInfo(jdbcConnectionInfo))); } @@ -113,9 +113,9 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept if (StringUtils.isNotEmpty(jdbcConnectionInfo.getCatalog())) { catalog = jdbcConnectionInfo.getCatalog(); - schema = param.getDataBase(); + schema = param.getDatabase(); } else { - catalog = param.getDataBase(); + catalog = param.getDatabase(); schema = StringUtils.isEmptyOrNullStr(jdbcConnectionInfo.getSchema()) ? null : jdbcConnectionInfo.getSchema(); } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnector.java index b24f6b0b0..b9621ae83 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnector.java @@ -104,7 +104,7 @@ private MongoClient getMongoClient(JdbcConnectionInfo jdbcConnectionInfo) { public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLException { ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - String dataBase = param.getDataBase(); + String dataBase = param.getDatabase(); JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); if (jdbcConnectionInfo == null) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConfigBuilder.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConfigBuilder.java index 7448dd4ef..4c5378deb 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConfigBuilder.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConfigBuilder.java @@ -16,10 +16,52 @@ */ package io.datavines.connector.plugin; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.datavines.common.param.form.PluginParams; +import io.datavines.common.param.form.Validate; import io.datavines.common.param.form.type.InputParam; +import lombok.extern.slf4j.Slf4j; +import java.util.ArrayList; +import java.util.List; + +@Slf4j public class MysqlConfigBuilder extends JdbcConfigBuilder { + @Override + public String buildErrorDataStorage(boolean isEn) { + List params = new ArrayList<>(); + params.add(getHostInput(isEn)); + params.add(getPortInput(isEn)); + if (getCatalogInput(isEn) != null) { + params.add(getCatalogInput(isEn)); + } + + params.add(getErrorDataStorageDatabaseInput(isEn)); + + if (getSchemaInput(isEn) != null) { + params.add(getSchemaInput(isEn)); + } + + params.add(getUserInput(isEn)); + params.add(getPasswordInput(isEn)); + params.add(getPropertiesInput(isEn)); + + ObjectMapper mapper = new ObjectMapper(); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + String result = null; + + try { + result = mapper.writeValueAsString(params); + } catch (JsonProcessingException e) { + log.error("json parse error : ", e); + } + + return result; + } + @Override protected InputParam getPropertiesInput(boolean isEn) { return getInputParam("properties", @@ -35,4 +77,12 @@ protected InputParam getDatabaseInput(boolean isEn) { isEn ? "please enter database" : "请填入数据库", 1, null, null); } + + protected InputParam getErrorDataStorageDatabaseInput(boolean isEn) { + return getInputParam("database", + isEn ? "database" : "数据库", + isEn ? "please enter database" : "请填入数据库", 1, + Validate.newBuilder().setRequired(true).setMessage(isEn ? "please enter database" : "请填入数据库").build(), + null); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnector.java index ad556e1f2..333775261 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnector.java @@ -30,6 +30,7 @@ public class MysqlConnector extends JdbcConnector { public MysqlConnector(DataSourceClient dataSourceClient) { super(dataSourceClient); } + @Override public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { return new MysqlDataSourceInfo(jdbcConnectionInfo); diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDialect.java index 81020fe69..804537a28 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDialect.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDialect.java @@ -44,7 +44,7 @@ public boolean invalidateItemCanOutputToSelf() { @Override public boolean supportToBeErrorDataStorage() { - return true; + return false; } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java index fe0b9f216..4a1ac870e 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java @@ -45,6 +45,6 @@ public boolean invalidateItemCanOutputToSelf() { @Override public boolean supportToBeErrorDataStorage() { - return true; + return false; } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConfigBuilder.java b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConfigBuilder.java new file mode 100644 index 000000000..3aee083dc --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConfigBuilder.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +import io.datavines.common.param.form.type.InputParam; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class StarRocksConfigBuilder extends JdbcConfigBuilder { + + @Override + protected InputParam getPropertiesInput(boolean isEn) { + return getInputParam("properties", + isEn ? "properties" : "参数", + isEn ? "please enter properties,like key=value&key1=value1" : "请填入参数，格式为key=value&key1=value1", 2, null, + "useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&useInformationSchema=true"); + } + + @Override + protected InputParam getDatabaseInput(boolean isEn) { + return getInputParam("database", + isEn ? "database" : "数据库", + isEn ? "please enter database" : "请填入数据库", 1, null, + null); + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnector.java index 39ad0cf34..e93343779 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnector.java @@ -16,11 +16,81 @@ */ package io.datavines.connector.plugin; +import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; +import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; +import io.datavines.common.datasource.jdbc.entity.TableInfo; +import io.datavines.common.datasource.jdbc.utils.JdbcDataSourceUtils; +import io.datavines.common.param.ConnectorResponse; +import io.datavines.common.param.GetTablesRequestParam; +import io.datavines.common.utils.JSONUtils; import io.datavines.connector.api.DataSourceClient; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; public class StarRocksConnector extends MysqlConnector { public StarRocksConnector(DataSourceClient dataSourceClient) { super(dataSourceClient); } + + @Override + public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { + return new StarRocksDataSourceInfo(jdbcConnectionInfo); + } + + @Override + public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLException { + ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); + String dataSourceParam = param.getDataSourceParam(); + + JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); + if (jdbcConnectionInfo == null) { + throw new SQLException("jdbc datasource param is no validate"); + } + + Connection connection = getConnection(dataSourceParam, jdbcConnectionInfo); + + List tableList = null; + ResultSet tables; + + try { + String schema = param.getDatabase(); + tableList = new ArrayList<>(); + tables = getMetadataTables(connection, schema); + + if (null == tables) { + return builder.result(tableList).build(); + } + + while (tables.next()) { + String name = tables.getString(TABLE_NAME); + if (!StringUtils.isEmpty(name)) { + String type = TABLE; + try { + type = tables.getString(TABLE_TYPE); + } catch (Exception e) { + // ignore + } + tableList.add(new TableInfo(schema, name, type, tables.getString("TABLE_COMMENT"))); + } + } + + } catch (Exception e) { + logger.error("get table list error: ", e); + } finally { + JdbcDataSourceUtils.releaseConnection(connection); + } + + return builder.result(tableList).build(); + } + + protected ResultSet getMetadataTables(Connection connection, String schema) throws SQLException { + java.sql.Statement stmt = connection.createStatement(); + return stmt.executeQuery("select TABLE_NAME, TABLE_TYPE, TABLE_COMMENT from information_schema.tables where TABLE_SCHEMA = '" + schema + "'"); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java index e167e2a09..ebfcb9c9c 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java @@ -42,6 +42,6 @@ public Executor getExecutor() { @Override public ConfigBuilder getConfigBuilder() { - return new MysqlConfigBuilder(); + return new StarRocksConfigBuilder(); } } diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricExecutionResult.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricExecutionResult.java index 038859258..7f999365a 100644 --- a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricExecutionResult.java +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricExecutionResult.java @@ -61,17 +61,52 @@ public MetricExecutionResult(Map dataMap) { throw new DataVinesException("data map is empty"); } - actualValue = Double.valueOf(String.valueOf(dataMap.get("actual_value"))); - expectedValue = Double.valueOf(String.valueOf(dataMap.get("expected_value"))); - resultFormula = String.valueOf(dataMap.get("result_formula")); - operator = String.valueOf(dataMap.get("operator")); - threshold = Double.valueOf(String.valueOf(dataMap.get("threshold"))); - metricName = String.valueOf(dataMap.get("metric_name")); - metricDimension = String.valueOf(dataMap.get("metric_dimension")); - metricType = String.valueOf(dataMap.get("metric_type")); - databaseName = String.valueOf(dataMap.get("database_name")); - tableName = String.valueOf(dataMap.get("table_name")); - columnName = String.valueOf(dataMap.get("column_name")); - expectedType = String.valueOf(dataMap.get("expected_type")); + if (dataMap.get("actual_value") != null) { + actualValue = Double.valueOf(String.valueOf(dataMap.get("actual_value")).trim()); + } + + if (dataMap.get("expected_value") != null) { + expectedValue = Double.valueOf(String.valueOf(dataMap.get("expected_value")).trim()); + } + + if (dataMap.get("result_formula") != null) { + resultFormula = String.valueOf(dataMap.get("result_formula")).trim(); + } + + if (dataMap.get("operator") != null) { + operator = String.valueOf(dataMap.get("operator")).trim(); + } + + if (dataMap.get("threshold") != null) { + threshold = Double.valueOf(String.valueOf(dataMap.get("threshold")).trim()); + } + + if (dataMap.get("metric_name") != null) { + metricName = String.valueOf(dataMap.get("metric_name")).trim(); + } + + if (dataMap.get("metric_dimension") != null) { + metricDimension = String.valueOf(dataMap.get("metric_dimension")).trim(); + } + + if (dataMap.get("metric_type") != null) { + metricType = String.valueOf(dataMap.get("metric_type")).trim(); + } + + if (dataMap.get("database_name") != null) { + databaseName = String.valueOf(dataMap.get("database_name")).trim(); + } + + if (dataMap.get("table_name") != null) { + tableName = String.valueOf(dataMap.get("table_name")).trim(); + } + + if (dataMap.get("column_name") != null) { + columnName = String.valueOf(dataMap.get("column_name")).trim(); + } + + if (dataMap.get("expected_type") != null) { + expectedType = String.valueOf(dataMap.get("expected_type")).trim(); + } } } \ No newline at end of file diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java index 07ad27338..197bdf6c8 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java @@ -34,6 +34,7 @@ import javax.validation.constraints.NotNull; import java.util.*; +import java.util.stream.Collectors; @Api(value = "metric", tags = "metric", produces = MediaType.APPLICATION_JSON_VALUE) @RestController @@ -150,19 +151,33 @@ public Object getMetricConfig(@PathVariable("name") String name) { } @ApiOperation(value = "get expected value list") - @GetMapping(value = "/expectedValue/list") - public Object getExpectedTypeList() { + @GetMapping(value = "/expectedValue/list/{type}") + public Object getExpectedTypeList(@PathVariable("type") String type) { Set expectedValueList = PluginLoader.getPluginLoader(ExpectedValue.class).getSupportedPlugins(); Set afterFilterSet = new HashSet<>(); - expectedValueList.forEach(it -> { - afterFilterSet.add(it - .replace("local_", "") - .replace("spark_","") - .replace("livy_","") - ); - }); + afterFilterSet = expectedValueList.stream() + .map(it ->it.replace("local_", "") + .replace("spark_","") + .replace("livy_","")) + .collect(Collectors.toSet()); List items = new ArrayList<>(); + JobType jobType = JobType.of(type); + if (jobType == null) { + throw new DataVinesServerException(type + "type is not validate"); + } + + switch (jobType) { + case DATA_QUALITY: + afterFilterSet = afterFilterSet.stream().filter(it -> !it.contains("target_table_total_rows")).collect(Collectors.toSet()); + break; + case DATA_RECONCILIATION: + // NOTHING TO DO + break; + default: + break; + } + afterFilterSet.forEach(it -> { ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class).getOrCreatePlugin("local_" + it); if (expectedValue != null) { diff --git a/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobExecutionVO.java b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobExecutionVO.java index 0c66a1829..9258a2ec4 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobExecutionVO.java +++ b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobExecutionVO.java @@ -20,6 +20,7 @@ import io.datavines.common.enums.ExecutionStatus; import io.datavines.common.enums.JobType; import io.datavines.core.utils.LanguageUtils; +import io.datavines.server.enums.DqJobExecutionState; import lombok.Data; import java.io.Serializable; @@ -46,6 +47,8 @@ public class JobExecutionVO implements Serializable { private ExecutionStatus status; + private DqJobExecutionState checkState; + @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") private LocalDateTime startTime; @@ -59,7 +62,11 @@ public String getJobType() { return LanguageUtils.isZhContext()? jobType.getZhDescription() : jobType.getDescription(); } + public String getCheckState() { + return LanguageUtils.isZhContext()? checkState.getZhDescription() : checkState.getDescription(); + } + public String getStatus() { - return status.getDescription(); + return LanguageUtils.isZhContext()? status.getZhDescription() : status.getDescription(); } } diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutorImpl.java b/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutorImpl.java index bb6ef9b67..d6e9f9240 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutorImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutorImpl.java @@ -284,7 +284,7 @@ private void executeFetchDatabase(String database) { GetTablesRequestParam getTablesRequestParam = new GetTablesRequestParam(); getTablesRequestParam.setType(dataSource.getType()); getTablesRequestParam.setDataSourceParam(dataSource.getParam()); - getTablesRequestParam.setDataBase(database); + getTablesRequestParam.setDatabase(database); ConnectorResponse connectorResponse = null; List tableListFromDataSource = new ArrayList<>(); Map tableInfoMap = new HashMap<>(); diff --git a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/operator/JobResultValidator.java b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/operator/JobResultValidator.java index 2804d5bd6..636b72b8a 100644 --- a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/operator/JobResultValidator.java +++ b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/operator/JobResultValidator.java @@ -88,12 +88,9 @@ public void operateDqExecuteResult(JobExecutionRequest jobExecutionRequest) { } if (!isSuccess) { - jobExternalService.updateJobExecutionStatus(jobExecutionId, ExecutionStatus.FAILURE); sendErrorEmail(jobExecutionId); } } - - } /** @@ -131,7 +128,7 @@ private void sendErrorEmail(Long jobExecutionId){ List errorJobExecutionResultList = jobExternalService.listErrorJobExecutionResultByJobExecutionId(jobExecution.getId()); boolean isEn = !LanguageUtils.isZhContext(); if (CollectionUtils.isNotEmpty(errorJobExecutionResultList)) { - for(JobExecutionResult errorJobExecutionResult : errorJobExecutionResultList){ + for (JobExecutionResult errorJobExecutionResult : errorJobExecutionResultList) { MetricExecutionResult metricExecutionResult = new MetricExecutionResult(); BeanUtils.copyProperties(errorJobExecutionResult, metricExecutionResult); List messages = new ArrayList<>(); @@ -148,7 +145,6 @@ private void sendErrorEmail(Long jobExecutionId){ } notificationClient.notify(message, config); } - } } diff --git a/datavines-server/src/main/java/io/datavines/server/enums/DqJobExecutionState.java b/datavines-server/src/main/java/io/datavines/server/enums/DqJobExecutionState.java index 1251d6f9d..2f20472f2 100644 --- a/datavines-server/src/main/java/io/datavines/server/enums/DqJobExecutionState.java +++ b/datavines-server/src/main/java/io/datavines/server/enums/DqJobExecutionState.java @@ -16,7 +16,9 @@ */ package io.datavines.server.enums; +import com.baomidou.mybatisplus.annotation.EnumValue; import com.fasterxml.jackson.annotation.JsonValue; +import lombok.Getter; import java.util.HashMap; import java.util.Map; @@ -40,23 +42,18 @@ public enum DqJobExecutionState { this.zhDescription = zhDescription; } + @Getter + @EnumValue private final int code; private final String description; + @Getter private final String zhDescription; - public int getCode() { - return code; - } - @JsonValue public String getDescription() { return description; } - public String getZhDescription() { - return zhDescription; - } - public String getDescription(boolean isEn) { return isEn ? description : zhDescription; } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java index 7e0d65662..7ae1dcf73 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java @@ -273,7 +273,7 @@ public Object getTableList(Long id, String database) throws DataVinesServerExcep GetTablesRequestParam param = new GetTablesRequestParam(); param.setType(dataSource.getType()); param.setDataSourceParam(dataSource.getParam()); - param.setDataBase(database); + param.setDatabase(database); Object result = null; ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(param.getType()); diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ErrorDataStorageServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ErrorDataStorageServiceImpl.java index 46a59b89e..e5bc65400 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ErrorDataStorageServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ErrorDataStorageServiceImpl.java @@ -137,7 +137,7 @@ public int deleteById(long id) { @Override public String getConfigJson(String type) { - return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(type).getConfigBuilder().build(!LanguageUtils.isZhContext()); + return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(type).getConfigBuilder().buildErrorDataStorage(!LanguageUtils.isZhContext()); } private boolean isErrorDataStorageExist(String name) { diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java index 81e1582af..6e02e6bef 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java @@ -342,25 +342,26 @@ public List getJobExecutionAggPie(JobExecutionDashboardPara return new ArrayList<>(); } items = items.stream().filter(it -> statusList.contains(it.getName())).collect(Collectors.toList()); - + Set resultStatusSet = items.stream().map(JobExecutionAggItem::getName).collect(Collectors.toSet()); + + for (String status : statusList) { + if (!resultStatusSet.contains(status)) { + JobExecutionAggItem item = new JobExecutionAggItem(); + item.setName(status); + item.setValue(0); + items.add(item); + } + } + items = items.stream().sorted(Comparator.comparing(JobExecutionAggItem::getName)).collect(Collectors.toList()); boolean isZh = LanguageUtils.isZhContext(); for (JobExecutionAggItem jobExecutionAggItem : items) { switch (jobExecutionAggItem.getName()) { - case "1": - if (isZh) { - jobExecutionAggItem.setName("执行中"); - } else { - jobExecutionAggItem.setName("Running"); - } - - break; case "6": if (isZh) { jobExecutionAggItem.setName("执行失败"); } else { jobExecutionAggItem.setName("Failure"); } - break; case "7": if (isZh) { @@ -368,15 +369,6 @@ public List getJobExecutionAggPie(JobExecutionDashboardPara } else { jobExecutionAggItem.setName("Success"); } - - break; - case "9": - if (isZh) { - jobExecutionAggItem.setName("停止"); - } else { - jobExecutionAggItem.setName("Kill"); - } - break; default: break; @@ -488,6 +480,4 @@ public JobExecutionTrendBar getJobExecutionTrendBar(JobExecutionDashboardParam d public JobExecutionStat getJobExecutionStat(Long jobId) { return baseMapper.getJobExecutionStat(jobId); } - - } diff --git a/datavines-server/src/main/resources/mapper/JobExecutionMapper.xml b/datavines-server/src/main/resources/mapper/JobExecutionMapper.xml index 90d81b01a..fc8930659 100644 --- a/datavines-server/src/main/resources/mapper/JobExecutionMapper.xml +++ b/datavines-server/src/main/resources/mapper/JobExecutionMapper.xml @@ -109,7 +109,7 @@