diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java index 2c119c75b..c54aaf733 100644 --- a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java @@ -29,7 +29,7 @@ public interface ConnectorFactory { Dialect getDialect(); - ConnectorParameterConverter getConnectorParameterConverter(); + ParameterConverter getConnectorParameterConverter(); Executor getExecutor(); @@ -42,4 +42,10 @@ public interface ConnectorFactory { StatementSplitter getStatementSplitter(); StatementParser getStatementParser(); + + MetricScript getMetricScript(); + + default Boolean showInFrontend() { + return true; + } } diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/MetricScript.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/MetricScript.java new file mode 100644 index 000000000..da4618ea6 --- /dev/null +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/MetricScript.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.api; + +public interface MetricScript { + + default String selectFromTable() { + return "select * from ${table}"; + } + + default String baseActualValue(String uniqueKey) { + return "select count(1) as actual_value_"+ uniqueKey +" from ${invalidate_items_table}"; + } + + default String baseDirectActualValue(String uniqueKey, String invalidateItemsSql) { + return "select count(1) as actual_value_"+ uniqueKey +" from ( " + invalidateItemsSql + " ) t"; + } + + default String avgActualValue(String uniqueKey) { + return "select avg(${column}) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String avgLengthActualValue(String uniqueKey) { + return "select avg(length(${column})) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String countDistinctActualValue(String uniqueKey) { + return "select count(distinct(${column})) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String histogramActualValue(String uniqueKey, String where) { + return "select concat(k, '\001', cast(count as varchar)) as actual_value_" + uniqueKey + " from (select if(${column} is null, 'NULL', cast(${column} as varchar)) as k, count(1) as count from ${table} " + where + " group by ${column} order by count desc limit 50) T"; + } + + default String maxActualValue(String uniqueKey) { + return "select max(${column}) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String maxLengthActualValue(String uniqueKey) { + return "select max(length(${column})) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String minActualValue(String uniqueKey) { + return "select min(${column}) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String minLengthActualValue(String uniqueKey) { + return "select min(length(${column})) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String stdDevActualValue(String uniqueKey) { + return "select stddev(${column}) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String sumActualValue(String uniqueKey) { + return "select sum(${column}) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String varianceActualValue(String uniqueKey) { + return "select variance(${column}) as actual_value_"+ uniqueKey +" from ${table}"; + } + + default String groupByHavingCountForUnique() { + return " group by ${column} having count(1) = 1 "; + } + + default String groupByHavingCountForDuplicate() { + return " group by ${column} having count(${column}) > 1 "; + } + + default String columnInEnums() { + return " (${column} in ( ${enum_list} )) "; + } + + default String columnNotInEnums() { + return " (${column} not in ( ${enum_list} ) or ${column} is null) "; + } + + default String columnNotNull() { + return " ${column} is not null "; + } + + default String columnIsNull() { + return " (${column} is null) "; + } + + default String columnLengthCompare() { + return " length(${column}) ${comparator} ${length} "; + } + + default String columnNotMatchRegex() { + return " ${column} not regexp '${regexp}' "; + } + + default String columnMatchRegex() { + return " ${column} regexp '${regexp}' "; + } + + default String columnGteMin() { + return " ${column} >= ${min} "; + } + + default String columnLteMax() { + return " ${column} <= ${max} "; + } + + default String columnIsBlank() { + return " (${column} is null or ${column} = '') "; + } + + default String timeBetweenWithFormat() { + return " (DATE_FORMAT(${column}, '${datetime_format}') <= DATE_FORMAT(${deadline_time}, '${datetime_format}') ) AND (DATE_FORMAT(${column}, '${datetime_format}') >= DATE_FORMAT(${begin_time}, '${datetime_format}')) "; + } + + default String dailyAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >=date_format(${data_time},'%Y-%m-%d')" + + " and data_time < date_add(date_format(${data_time},'%Y-%m-%d'), interval 1 DAY)" + + " and unique_code = ${unique_code}"; + } + + default String last7DayAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >= date_sub(date_format(${data_time},'%Y-%m-%d'),interval 7 DAY)" + + " and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY) and unique_code = ${unique_code}"; + } + + default String last30DayAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >= date_sub(date_format(${data_time},'%Y-%m-%d'),interval 30 DAY)" + + " and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY) and unique_code = ${unique_code}"; + } + + default String monthlyAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >= date_format(curdate(), '%Y-%m-01') and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY)" + + " and unique_code = ${unique_code}"; + } + + default String weeklyAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >= date_sub(${data_time},interval weekday(${data_time}) + 0 day)" + + " and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY) and unique_code = ${unique_code}"; + } +} diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorParameterConverter.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ParameterConverter.java similarity index 96% rename from datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorParameterConverter.java rename to datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ParameterConverter.java index d1dd2ceae..9aee38ce2 100644 --- a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ParameterConverter.java @@ -22,7 +22,7 @@ import static io.datavines.common.ConfigConstants.*; -public interface ConnectorParameterConverter { +public interface ParameterConverter { Map converter(Map parameter); diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-all/pom.xml b/datavines-connector/datavines-connector-plugins/datavines-connector-all/pom.xml index d3ba62d2e..98522bc31 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-all/pom.xml +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-all/pom.xml @@ -158,6 +158,12 @@ ${project.version} + + io.datavines + datavines-connector-spark + ${project.version} + + \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorFactory.java index 6444c4bcf..fa41e34de 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorFactory.java @@ -16,16 +16,13 @@ */ package io.datavines.connector.plugin; -import io.datavines.connector.api.Connector; -import io.datavines.connector.api.ConnectorParameterConverter; -import io.datavines.connector.api.Dialect; -import io.datavines.connector.api.Executor; +import io.datavines.connector.api.*; public class ClickHouseConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new ClickHouseConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new ClickHouseParameterConverter(); } @Override @@ -42,4 +39,9 @@ public Connector getConnector() { public Executor getExecutor() { return new ClickHouseExecutor(getDataSourceClient()); } + + @Override + public MetricScript getMetricScript() { + return new ClickHouseMetricScript(); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseDialect.java index 6c8cef7f5..df420d248 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseDialect.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseDialect.java @@ -16,24 +16,10 @@ */ package io.datavines.connector.plugin; -import java.util.Map; - -import static io.datavines.common.ConfigConstants.STD_DEV_KEY; -import static io.datavines.common.ConfigConstants.VARIANCE_KEY; - public class ClickHouseDialect extends JdbcDialect { @Override public String getDriver() { return "ru.yandex.clickhouse.ClickHouseDriver"; } - - @Override - public Map getDialectKeyMap() { - super.getDialectKeyMap(); - dialectKeyMap.put(STD_DEV_KEY, "STDDEV_POP"); - dialectKeyMap.put(VARIANCE_KEY, "VAR_POP"); - return dialectKeyMap; - - } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseMetricScript.java b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseMetricScript.java new file mode 100644 index 000000000..a5c034492 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseMetricScript.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +public class ClickHouseMetricScript extends JdbcMetricScript { + + @Override + public String stdDevActualValue(String uniqueKey) { + return "select STDDEV_POP(${column}) as actual_value_"+ uniqueKey +" from ${table}"; + } + + @Override + public String varianceActualValue(String uniqueKey) { + return "select VAR_POP(${column}) as actual_value_"+ uniqueKey +" from ${table}"; + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseParameterConverter.java similarity index 92% rename from datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseParameterConverter.java index c1a24a860..08064d092 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseParameterConverter.java @@ -20,7 +20,7 @@ import static io.datavines.common.ConfigConstants.*; -public class ClickHouseConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class ClickHouseParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorFactory.java index c9dea5c99..a79e12243 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorFactory.java @@ -21,8 +21,8 @@ public class DatabendConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new DatabendConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new DatabendParameterConverter(); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendParameterConverter.java similarity index 94% rename from datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendParameterConverter.java index 5010207d1..360519cbc 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendParameterConverter.java @@ -22,7 +22,7 @@ import static io.datavines.common.ConfigConstants.*; -public class DatabendConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class DatabendParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorFactory.java index efb90e497..579c46f46 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorFactory.java @@ -31,8 +31,8 @@ public Dialect getDialect() { } @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new DmConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new DmParameterConverter(); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmParameterConverter.java similarity index 94% rename from datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmParameterConverter.java index b093456e3..9e4c0dc8f 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmParameterConverter.java @@ -21,7 +21,7 @@ import static io.datavines.common.ConfigConstants.*; import static io.datavines.common.ConfigConstants.PROPERTIES; -public class DmConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class DmParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { // in dm jdbc url, the database is not need. diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorFactory.java index fb9ec5894..cce578989 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorFactory.java @@ -21,8 +21,8 @@ public class DorisConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new DorisConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new DorisParameterConverter(); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisParameterConverter.java similarity index 95% rename from datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisParameterConverter.java index 4d13eb905..322b8d25d 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisParameterConverter.java @@ -22,7 +22,7 @@ import static io.datavines.common.ConfigConstants.*; -public class DorisConnectorParameterConverter extends MysqlConnectorParameterConverter { +public class DorisParameterConverter extends MysqlParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java index 6d7ddcb7a..f0a782cee 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java @@ -41,8 +41,8 @@ public Dialect getDialect() { } @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new FileConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new FileParameterConverter(); } @Override @@ -74,4 +74,14 @@ public StatementSplitter getStatementSplitter() { public StatementParser getStatementParser() { return null; } + + @Override + public MetricScript getMetricScript() { + return null; + } + + @Override + public Boolean showInFrontend() { + return false; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileParameterConverter.java similarity index 86% rename from datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileParameterConverter.java index 62a997d57..3558b3389 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileParameterConverter.java @@ -16,11 +16,11 @@ */ package io.datavines.connector.plugin; -import io.datavines.connector.api.ConnectorParameterConverter; +import io.datavines.connector.api.ParameterConverter; import java.util.Map; -public class FileConnectorParameterConverter implements ConnectorParameterConverter { +public class FileParameterConverter implements ParameterConverter { @Override public Map converter(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorFactory.java index f06a3b6af..fa67d7d7d 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorFactory.java @@ -21,8 +21,8 @@ public class HiveConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new HiveConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new HiveParameterConverter(); } @Override @@ -44,4 +44,9 @@ public Executor getExecutor() { public ConfigBuilder getConfigBuilder() { return new HiveConfigBuilder(); } + + @Override + public MetricScript getMetricScript() { + return new HiveMetricScript(); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveDialect.java index fe0c814dd..dc1fac815 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveDialect.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveDialect.java @@ -28,18 +28,9 @@ import java.util.List; import java.util.Map; -import static io.datavines.common.ConfigConstants.STRING_TYPE; - @Slf4j public class HiveDialect extends JdbcDialect { - @Override - public Map getDialectKeyMap() { - super.getDialectKeyMap(); - dialectKeyMap.put(STRING_TYPE, "string"); - return dialectKeyMap; - } - @Override public String getDriver() { return "org.apache.hive.jdbc.HiveDriver"; diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveMetricScript.java b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveMetricScript.java new file mode 100644 index 000000000..250758870 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveMetricScript.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +public class HiveMetricScript extends JdbcMetricScript { + + @Override + public String histogramActualValue(String uniqueKey, String where) { + return "select concat(k, '\001', cast(count as string)) as actual_value_" + uniqueKey + + " from (select if(${column} is null, 'NULL', cast(${column} as string)) as k, count(1) as count from ${table} " + + where + " group by ${column} order by count desc limit 50) T"; + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveParameterConverter.java similarity index 95% rename from datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveParameterConverter.java index b55be3c06..2dc99a80e 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveParameterConverter.java @@ -22,7 +22,7 @@ import static io.datavines.common.ConfigConstants.*; -public class HiveConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class HiveParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorFactory.java index fede31317..ac2515d74 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorFactory.java @@ -17,15 +17,15 @@ package io.datavines.connector.plugin; import io.datavines.connector.api.Connector; -import io.datavines.connector.api.ConnectorParameterConverter; +import io.datavines.connector.api.ParameterConverter; import io.datavines.connector.api.Dialect; import io.datavines.connector.api.Executor; public class ImpalaConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new ImpalaConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new ImpalaParameterConverter(); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaParameterConverter.java similarity index 95% rename from datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaParameterConverter.java index 7a8b02166..5b2c089ae 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaParameterConverter.java @@ -22,7 +22,7 @@ import static io.datavines.common.ConfigConstants.*; -public class ImpalaConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class ImpalaParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/AbstractJdbcConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/AbstractJdbcConnectorFactory.java index 7ab6a9514..5d2f300c2 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/AbstractJdbcConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/AbstractJdbcConnectorFactory.java @@ -54,4 +54,9 @@ public StatementSplitter getStatementSplitter() { public StatementParser getStatementParser() { return new DefaultStatementParser(); } + + @Override + public MetricScript getMetricScript() { + return new JdbcMetricScript(); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementSplitter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementSplitter.java index f2404f08c..ab19404aa 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementSplitter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementSplitter.java @@ -26,8 +26,6 @@ public class DefaultStatementSplitter implements StatementSplitter { @Override public List splitStatements(String body) { - - String DELIMITER = ";"; - return StringUtils.isEmpty(body) ? null : Arrays.asList(body.split(DELIMITER)); + return StringUtils.isEmpty(body) ? null : Arrays.asList(body.split(";")); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcDialect.java index f9f6951ad..4f40f633d 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcDialect.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcDialect.java @@ -33,22 +33,6 @@ public abstract class JdbcDialect implements Dialect { - protected final HashMap dialectKeyMap = new HashMap<>(); - - @Override - public Map getDialectKeyMap() { - dialectKeyMap.put(REGEX_KEY, "${column} regexp '${regexp}'"); - dialectKeyMap.put(NOT_REGEX_KEY, "${column} not regexp '${regexp}'"); - dialectKeyMap.put(STRING_TYPE, "varchar"); - dialectKeyMap.put(IF_FUNCTION_KEY, "if"); - dialectKeyMap.put(LIMIT_TOP_50_KEY, " limit 50"); - dialectKeyMap.put(LENGTH_KEY, "length(${column})"); - dialectKeyMap.put(IF_CASE_KEY, "if(${column} is null, 'NULL', cast(${column} as ${string_type}))"); - dialectKeyMap.put(STD_DEV_KEY, "stddev"); - dialectKeyMap.put(VARIANCE_KEY, "variance"); - return dialectKeyMap; - } - @Override public String getColumnPrefix() { return "`"; diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcMetricScript.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcMetricScript.java new file mode 100644 index 000000000..5b2773cf1 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcMetricScript.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +import io.datavines.connector.api.MetricScript; + +public class JdbcMetricScript implements MetricScript { + +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcParameterConverter.java similarity index 89% rename from datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcParameterConverter.java index a8d84b0fe..728679ec5 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcParameterConverter.java @@ -19,11 +19,10 @@ import java.util.HashMap; import java.util.Map; -import io.datavines.common.utils.StringUtils; -import io.datavines.connector.api.ConnectorParameterConverter; +import io.datavines.connector.api.ParameterConverter; import static io.datavines.common.ConfigConstants.*; -public abstract class JdbcConnectorParameterConverter implements ConnectorParameterConverter { +public abstract class JdbcParameterConverter implements ParameterConverter { @Override public Map converter(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorFactory.java index de1aa8009..78591eda8 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorFactory.java @@ -26,8 +26,8 @@ public String getCategory() { } @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new MongodbConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new MongodbParameterConverter(); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbParameterConverter.java similarity index 97% rename from datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbParameterConverter.java index 6824a35e8..3ae33a013 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbParameterConverter.java @@ -24,7 +24,7 @@ import static io.datavines.common.ConfigConstants.*; -public class MongodbConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class MongodbParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorFactory.java index 4a51474e5..303853ef2 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorFactory.java @@ -21,8 +21,8 @@ public class MysqlConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new MysqlConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new MysqlParameterConverter(); } @Override @@ -44,4 +44,9 @@ public Executor getExecutor() { public ConfigBuilder getConfigBuilder() { return new MysqlConfigBuilder(); } + + @Override + public MetricScript getMetricScript() { + return new MysqlMetricScript(); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlDialect.java index 2d1150f2b..a713f5463 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlDialect.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlDialect.java @@ -22,13 +22,6 @@ public class MysqlDialect extends JdbcDialect { - @Override - public Map getDialectKeyMap() { - super.getDialectKeyMap(); - dialectKeyMap.put(STRING_TYPE, "char"); - return dialectKeyMap; - } - @Override public String getDriver() { return "com.mysql.cj.jdbc.Driver"; diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlMetricScript.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlMetricScript.java new file mode 100644 index 000000000..c274380ff --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlMetricScript.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +public class MysqlMetricScript extends JdbcMetricScript { + + @Override + public String histogramActualValue(String uniqueKey, String where) { + return "select concat(k, '\001', cast(count as char)) as actual_value_" + + uniqueKey + " from (select if(${column} is null, 'NULL', cast(${column} as char)) as k, count(1) as count from ${table} " + + where + " group by ${column} order by count desc limit 50) T"; + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlParameterConverter.java similarity index 94% rename from datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlParameterConverter.java index 8e5911214..0910584ad 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlParameterConverter.java @@ -21,7 +21,7 @@ import io.datavines.common.utils.StringUtils; import static io.datavines.common.ConfigConstants.*; -public class MysqlConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class MysqlParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorFactory.java index fa2c3b2e3..cfc6ffdb0 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorFactory.java @@ -16,16 +16,13 @@ */ package io.datavines.connector.plugin; -import io.datavines.connector.api.ConnectorParameterConverter; -import io.datavines.connector.api.Dialect; -import io.datavines.connector.api.Connector; -import io.datavines.connector.api.Executor; -import io.datavines.connector.api.ConfigBuilder; -import io.datavines.connector.api.TypeConverter; +import io.datavines.connector.api.*; + public class OracleConnectorFactory extends AbstractJdbcConnectorFactory{ + @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new OracleConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new OracleParameterConverter(); } @Override @@ -52,4 +49,9 @@ public ConfigBuilder getConfigBuilder() { public TypeConverter getTypeConverter() { return new OracleTypeConverter(); } + + @Override + public MetricScript getMetricScript() { + return new OracleMetricScript(); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDialect.java index 29ffdb358..30318d9ba 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDialect.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDialect.java @@ -18,21 +18,7 @@ import io.datavines.common.utils.StringUtils; -import java.util.Map; - -import static io.datavines.common.ConfigConstants.*; - public class OracleDialect extends JdbcDialect{ - @Override - public Map getDialectKeyMap() { - super.getDialectKeyMap(); - dialectKeyMap.put(STRING_TYPE, "VARCHAR2"); - dialectKeyMap.put(LIMIT_TOP_50_KEY, " rownum <= 50"); - dialectKeyMap.put(REGEX_KEY, "regexp_like(${column},'${regexp}')"); - dialectKeyMap.put(NOT_REGEX_KEY, "not regexp_like(${column},'${regexp}')"); - dialectKeyMap.put(IF_CASE_KEY, "case when ${column} is null then 'NULL' else ${column}||'' end "); - return dialectKeyMap; - } @Override public String getDriver() { @@ -44,21 +30,6 @@ public boolean invalidateItemCanOutputToSelf() { return true; } - @Override - public boolean supportToBeErrorDataStorage() { - return false; - } - - @Override - public String getCreateTableAsSelectStatement(String srcTable, String targetDatabase, String targetTable) { - return String.format("CREATE TABLE %s.%s AS SELECT * FROM %s", quoteIdentifier(targetDatabase), quoteIdentifier(targetTable), quoteIdentifier(srcTable)); - } - - @Override - public String quoteIdentifier(String column) { - return column; - } - @Override public String getFullQualifiedTableName(String database, String schema, String table,boolean needQuote) { table = quoteIdentifier(table); diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleMetricScript.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleMetricScript.java new file mode 100644 index 000000000..f549d298a --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleMetricScript.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +public class OracleMetricScript extends JdbcMetricScript { + + @Override + public String histogramActualValue(String uniqueKey, String where) { + return "select concat(concat(k, '\001'), count) as actual_value_" + uniqueKey + + " from (select case when ${column} is null then 'NULL' else ${column}||'' end as k, count(1) as count from ${table} " + + where + " group by ${column} order by count desc ) T where rownum <= 50 "; + } + + @Override + public String columnMatchRegex() { + return " regexp_like(${column},'${regexp}') "; + } + + @Override + public String columnNotMatchRegex() { + return " not regexp_like(${column},'${regexp}') "; + } + + @Override + public String timeBetweenWithFormat() { + return " (to_char(${column}, '${datetime_format}') <= to_char(${deadline_time}, '${datetime_format}') ) AND (to_char(${column}, '${datetime_format}') >= to_char(${begin_time}, '${datetime_format}')) "; + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleParameterConverter.java similarity index 94% rename from datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleParameterConverter.java index 7d30e5e94..2c131e27e 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleParameterConverter.java @@ -22,7 +22,7 @@ import static io.datavines.common.ConfigConstants.*; -public class OracleConnectorParameterConverter extends JdbcConnectorParameterConverter{ +public class OracleParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorFactory.java index c2dda31b9..45d8fcd27 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorFactory.java @@ -21,8 +21,8 @@ public class PostgreSqlConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new PostgreSqlConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new PostgreSqlParameterConverter(); } @Override @@ -49,4 +49,9 @@ public ConfigBuilder getConfigBuilder() { public TypeConverter getTypeConverter() { return new PostgreSqlTypeConverter(); } + + @Override + public MetricScript getMetricScript() { + return new PostgreSqlMetricScript(); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDialect.java index b16f969da..e13aa9422 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDialect.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDialect.java @@ -16,22 +16,8 @@ */ package io.datavines.connector.plugin; -import java.util.Map; - -import static io.datavines.common.ConfigConstants.*; - public class PostgreSqlDialect extends JdbcDialect { - @Override - public Map getDialectKeyMap() { - super.getDialectKeyMap(); - dialectKeyMap.put(REGEX_KEY, "${column} ~ '${regexp}'"); - dialectKeyMap.put(NOT_REGEX_KEY, "${column} !~ '${regexp}'"); - dialectKeyMap.put(LENGTH_KEY, "length(${column}::text)"); - dialectKeyMap.put(IF_CASE_KEY, "case when ${column} is null then 'NULL' else cast(${column} as ${string_type}) end "); - return dialectKeyMap; - } - @Override public String getDriver() { return "org.postgresql.Driver"; diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlMetricScript.java b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlMetricScript.java new file mode 100644 index 000000000..681b7d59c --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlMetricScript.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +public class PostgreSqlMetricScript extends JdbcMetricScript { + + @Override + public String maxLengthActualValue(String uniqueKey) { + return "select max(length(${column}::text)) as actual_value_"+ uniqueKey +" from ${table}"; + } + + @Override + public String minLengthActualValue(String uniqueKey) { + return "select min(length(${column}::text)) as actual_value_"+ uniqueKey +" from ${table}"; + } + + @Override + public String columnLengthCompare() { + return " length(${column}::text) ${comparator} ${length} "; + } + + @Override + public String columnMatchRegex() { + return " ${column} ~ '${regexp}' "; + } + + @Override + public String columnNotMatchRegex() { + return " ${column} !~ '${regexp}' "; + } + + @Override + public String avgLengthActualValue(String uniqueKey) { + return "select avg(length(${column}::text)) as actual_value_"+ uniqueKey +" from ${table}"; + } + + @Override + public String histogramActualValue(String uniqueKey, String where) { + return "select concat(k, '\001', cast(count as varchar)) as actual_value_" + + uniqueKey + " from (select case when ${column} is null then 'NULL' else cast(${column} as varchar) end as k, count(1) as count from ${table} " + + where + " group by ${column} order by count desc limit 50) T"; + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlParameterConverter.java similarity index 93% rename from datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlParameterConverter.java index 1df101379..1498894d7 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlParameterConverter.java @@ -22,7 +22,7 @@ import static io.datavines.common.ConfigConstants.*; -public class PostgreSqlConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class PostgreSqlParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorFactory.java index 113dc31a0..f28fba1a0 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorFactory.java @@ -21,8 +21,8 @@ public class PrestoConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new PrestoConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new PrestoParameterConverter(); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoParameterConverter.java similarity index 91% rename from datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoParameterConverter.java index 1225b70a6..d2bd1a62e 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoParameterConverter.java @@ -17,14 +17,12 @@ package io.datavines.connector.plugin; import io.datavines.common.utils.StringUtils; -import io.datavines.connector.api.ConnectorParameterConverter; -import java.util.HashMap; import java.util.Map; import static io.datavines.common.ConfigConstants.*; -public class PrestoConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class PrestoParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/pom.xml b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/pom.xml new file mode 100644 index 000000000..6dc136e24 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/pom.xml @@ -0,0 +1,41 @@ + + + + 4.0.0 + + io.datavines + datavines-connector-plugins + 1.0.0-SNAPSHOT + + + datavines-connector-spark + + + + io.datavines + datavines-connector-jdbc + ${project.version} + + + + + \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkConnectorFactory.java new file mode 100644 index 000000000..f13d2ef43 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkConnectorFactory.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +import io.datavines.connector.api.*; + +public class SparkConnectorFactory extends AbstractJdbcConnectorFactory { + + @Override + public Connector getConnector() { + return null; + } + + @Override + public Dialect getDialect() { + return new SparkDialect(); + } + + @Override + public ParameterConverter getConnectorParameterConverter() { + return null; + } + + @Override + public Executor getExecutor() { + return null; + } + + @Override + public MetricScript getMetricScript() { + return new SparkMetricScript(); + } + + @Override + public Boolean showInFrontend() { + return false; + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkDialect.java new file mode 100644 index 000000000..1eb869369 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkDialect.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +import io.datavines.connector.api.Dialect; +import io.datavines.connector.api.entity.ResultList; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class SparkDialect implements Dialect { + + @Override + public String getDriver() { + return ""; + } + + @Override + public String getColumnPrefix() { + return ""; + } + + @Override + public String getColumnSuffix() { + return ""; + } + + @Override + public List getExcludeDatabases() { + return new ArrayList<>(); + } + + @Override + public String getErrorDataScript(Map configMap) { + return ""; + } + + @Override + public String getValidateResultDataScript(Map configMap) { + return ""; + } + + @Override + public ResultList getPageFromResultSet(Statement sourceConnectionStatement, ResultSet rs, String sourceTable, int start, int end) throws SQLException { + return null; + } +} \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkMetricScript.java b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkMetricScript.java new file mode 100644 index 000000000..4455ca9cc --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkMetricScript.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +public class SparkMetricScript extends JdbcMetricScript { + + @Override + public String histogramActualValue(String uniqueKey, String where) { + return "select concat(k, '\\001', cast(count as string)) as actual_value_" + uniqueKey + + " from (select if(${column} is null, 'NULL', cast(${column} as string)) as k, count(1) as count from ${table} " + + where + " group by ${column} order by count desc limit 50) T "; + } + + @Override + public String columnMatchRegex() { + return " regexp(${column}, ${regex}) "; + } + + @Override + public String columnNotMatchRegex() { + return " !regexp(${column}, ${regex}) "; + } + + @Override + public String dailyAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >=date_format(${data_time}, 'yyyy-MM-dd')" + + " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; + } + + @Override + public String last7DayAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >= date_add(date_format(${data_time}, 'yyyy-MM-dd'),-7)" + + " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; + } + + @Override + public String last30DayAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >= date_add(date_format(${data_time}, 'yyyy-MM-dd'),-30)" + + " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; + } + + @Override + public String monthlyAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >= date_format(${data_time}, 'yyyy-MM-01')" + + " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; + } + + @Override + public String weeklyAvg(String uniqueKey) { + return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + + " from dv_actual_values where data_time >= date_sub(date_format(${data_time},'yyyy-MM-dd'), (7- datediff(next_day(date_format(${data_time}, 'yyyy-MM-dd'),'Sunday'),date_format(${data_time}, 'yyyy-MM-dd')))-1)" + + " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..e46f3bf58 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +spark=io.datavines.connector.plugin.SparkConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorFactory.java index fdb578065..1efee656d 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorFactory.java @@ -21,8 +21,8 @@ public class SqlServerConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new SqlServerConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new SqlServerParameterConverter(); } @Override @@ -44,4 +44,9 @@ public Executor getExecutor() { public ConfigBuilder getConfigBuilder() { return new SqlServerConfigBuilder(); } + + @Override + public MetricScript getMetricScript() { + return new SqlServerMetricScript(); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java index 4a1ac870e..0d732d6b2 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java @@ -16,23 +16,8 @@ */ package io.datavines.connector.plugin; -import java.util.Map; - -import static io.datavines.common.ConfigConstants.*; - public class SqlServerDialect extends JdbcDialect { - @Override - public Map getDialectKeyMap() { - super.getDialectKeyMap(); - dialectKeyMap.put(STRING_TYPE, "char"); - dialectKeyMap.put(LENGTH_KEY, "len(${column})"); - dialectKeyMap.put(IF_FUNCTION_KEY, "iif"); - dialectKeyMap.put(LIMIT_TOP_50_KEY, " OFFSET 0 ROWS FETCH NEXT 50 ROWS ONLY"); - dialectKeyMap.put(IF_CASE_KEY, "iif(${column} is null, 'NULL', cast(${column} as ${string_type}))"); - return dialectKeyMap; - } - @Override public String getDriver() { return "com.microsoft.sqlserver.jdbc.SQLServerDriver"; @@ -43,8 +28,4 @@ public boolean invalidateItemCanOutputToSelf() { return true; } - @Override - public boolean supportToBeErrorDataStorage() { - return false; - } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerMetricScript.java b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerMetricScript.java new file mode 100644 index 000000000..4db9cfa6c --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerMetricScript.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +public class SqlServerMetricScript extends JdbcMetricScript { + + @Override + public String maxLengthActualValue(String uniqueKey) { + return "select max(length(${column}::text)) as actual_value_"+ uniqueKey +" from ${table}"; + } + + @Override + public String minLengthActualValue(String uniqueKey) { + return "select min(length(${column}::text)) as actual_value_"+ uniqueKey +" from ${table}"; + } + + @Override + public String columnLengthCompare() { + return " length(${column}::text) ${comparator} ${length} "; + } + + @Override + public String avgLengthActualValue(String uniqueKey) { + return "select avg(length(${column}::text)) as actual_value_"+ uniqueKey +" from ${table}"; + } + + @Override + public String histogramActualValue(String uniqueKey, String where) { + return "select concat(k, '\001', cast(count as char)) as actual_value_" + + uniqueKey + " from (select iif(${column} is null, 'NULL', cast(${column} as char)) as k, count(1) as count from ${table} " + + where + " group by ${column} order by count desc OFFSET 0 ROWS FETCH NEXT 50 ROWS ONLY) T"; + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerParameterConverter.java similarity index 93% rename from datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerParameterConverter.java index 8adab62cd..7c6877816 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerParameterConverter.java @@ -21,7 +21,7 @@ import io.datavines.common.utils.StringUtils; import static io.datavines.common.ConfigConstants.*; -public class SqlServerConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class SqlServerParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java index ebfcb9c9c..600d205a6 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java @@ -21,8 +21,8 @@ public class StarRocksConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new StarRocksConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new StarRocksParameterConverter(); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksParameterConverter.java similarity index 95% rename from datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksParameterConverter.java index 27cab8857..4d0849d6b 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksParameterConverter.java @@ -22,7 +22,7 @@ import static io.datavines.common.ConfigConstants.*; -public class StarRocksConnectorParameterConverter extends MysqlConnectorParameterConverter { +public class StarRocksParameterConverter extends MysqlParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorFactory.java index 92139d0fd..4d71bf825 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorFactory.java @@ -21,8 +21,8 @@ public class TrinoConnectorFactory extends AbstractJdbcConnectorFactory { @Override - public ConnectorParameterConverter getConnectorParameterConverter() { - return new TrinoConnectorParameterConverter(); + public ParameterConverter getConnectorParameterConverter() { + return new TrinoParameterConverter(); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoParameterConverter.java similarity index 91% rename from datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorParameterConverter.java rename to datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoParameterConverter.java index 6c0366504..ce1040f45 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoParameterConverter.java @@ -17,14 +17,12 @@ package io.datavines.connector.plugin; import io.datavines.common.utils.StringUtils; -import io.datavines.connector.api.ConnectorParameterConverter; -import java.util.HashMap; import java.util.Map; import static io.datavines.common.ConfigConstants.*; -public class TrinoConnectorParameterConverter extends JdbcConnectorParameterConverter { +public class TrinoParameterConverter extends JdbcParameterConverter { @Override protected String getUrl(Map parameter) { diff --git a/datavines-connector/datavines-connector-plugins/pom.xml b/datavines-connector/datavines-connector-plugins/pom.xml index 2b2ec91f9..53413de33 100644 --- a/datavines-connector/datavines-connector-plugins/pom.xml +++ b/datavines-connector/datavines-connector-plugins/pom.xml @@ -48,6 +48,7 @@ datavines-connector-sqlserver datavines-connector-dm datavines-connector-oracle + datavines-connector-spark diff --git a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/entity/ConfigRequest.java b/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/entity/ConfigRequest.java deleted file mode 100644 index b5da83694..000000000 --- a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/entity/ConfigRequest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.engine.config.entity; - -import io.datavines.common.entity.JobExecutionInfo; - -import java.util.Map; - -public class ConfigRequest { - private Map inputParameter; - private JobExecutionInfo jobExecutionInfo; - - public ConfigRequest() { - } - - public ConfigRequest(Map inputParameter, JobExecutionInfo jobExecutionInfo) { - this.inputParameter = inputParameter; - this.jobExecutionInfo = jobExecutionInfo; - } - - public Map getInputParameter() { - return inputParameter; - } - - public void setInputParameter(Map inputParameter) { - this.inputParameter = inputParameter; - } - - public JobExecutionInfo getJobExecutionInfo() { - return jobExecutionInfo; - } - - public void setJobExecutionInfo(JobExecutionInfo jobExecutionInfo) { - this.jobExecutionInfo = jobExecutionInfo; - } -} diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java index b6253e42f..4d502d05f 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java @@ -94,7 +94,7 @@ protected List getSourceConfigs() throws DataVinesException { connectorParameterMap.put(PRE_SQL, metricInputParameter.get(PRE_SQL)); connectorParameterMap.put(POST_SQL, metricInputParameter.get(POST_SQL)); - metricInputParameter.putAll(connectorFactory.getDialect().getDialectKeyMap()); +// metricInputParameter.putAll(connectorFactory.getDialect().getDialectKeyMap()); metricInputParameter.put(SRC_CONNECTOR_TYPE, connectorParameter.getType()); metricInputParameter.put(TABLE, table); @@ -146,7 +146,7 @@ protected List getSourceConfigs() throws DataVinesException { connectorParameterMap.put(SRC_CONNECTOR_TYPE, connectorParameter2.getType()); connectorParameterMap.put(PRE_SQL, metricInputParameter.get(PRE_SQL)); connectorParameterMap.put(POST_SQL, metricInputParameter.get(POST_SQL)); - metricInputParameter.putAll(connectorFactory.getDialect().getDialectKeyMap()); +// metricInputParameter.putAll(connectorFactory.getDialect().getDialectKeyMap()); metricInputParameter.put(SRC_CONNECTOR_TYPE, connectorParameter2.getType()); metricInputParameter.put(TABLE2, table); boolean invalidateItemCanOutput = Boolean.parseBoolean(metricInputParameter.get(INVALIDATE_ITEM_CAN_OUTPUT)); diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/BaseDataSinkExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/BaseDataSinkExecutor.java index 31801506d..1d7c9c3fe 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/BaseDataSinkExecutor.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/BaseDataSinkExecutor.java @@ -19,17 +19,19 @@ import io.datavines.common.config.Config; import io.datavines.common.utils.ParameterUtils; import io.datavines.common.utils.StringUtils; +import io.datavines.common.utils.ThreadUtils; import io.datavines.engine.local.api.LocalRuntimeEnvironment; import io.datavines.engine.local.api.utils.LoggerFactory; import io.datavines.connector.plugin.utils.SqlUtils; + import org.slf4j.Logger; import java.sql.SQLException; import java.sql.Statement; import java.util.Map; -import static io.datavines.common.ConfigConstants.INVALIDATE_ITEMS_TABLE; -import static io.datavines.common.ConfigConstants.SQL; +import static io.datavines.common.ConfigConstants.*; +import static io.datavines.common.ConfigConstants.TABLE; import static io.datavines.engine.api.EngineConstants.PLUGIN_TYPE; import static io.datavines.engine.local.connector.BaseJdbcSink.CREATE_TABLE_SQL; import static io.datavines.engine.local.connector.BaseJdbcSink.SINK_TABLE_NAME; @@ -84,15 +86,25 @@ private void executeInsert(String sql, LocalRuntimeEnvironment env) throws SQLEx } private boolean checkTableExist(LocalRuntimeEnvironment env, String tableName) throws SQLException { - //定义一个变量标示 + int retryTimes = 3; boolean flag = false ; - //一个查询该表所有的语句。 - String sql = "SELECT 1 FROM "+ tableName ; - try (Statement statement = env.getMetadataConnection().getConnection().createStatement()) { - statement.executeQuery(sql); - flag = true; - } catch (Exception e) { - log.warn("table {} is not exist", tableName); + + while (retryTimes > 0 && !flag) { + try { + log.info("start check table {} exists", config.getString(TABLE)); + //一个查询该表所有的语句。 + String sql = "SELECT 1 FROM "+ tableName ; + try (Statement statement = env.getMetadataConnection().getConnection().createStatement()) { + statement.executeQuery(sql); + flag = true; + } catch (Exception e) { + log.warn("table {} is not exist", tableName); + } + } catch (Exception e) { + log.error("check table {} exists error :", config.getString(TABLE), e); + retryTimes--; + ThreadUtils.sleep(2000); + } } return flag; } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/pom.xml new file mode 100644 index 000000000..861ed46ed --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + + io.datavines + datavines-metric-expected-plugins + 1.0.0-SNAPSHOT + + + datavines-metric-expected-base + + + 17 + 17 + UTF-8 + + + + + io.datavines + datavines-metric-api + ${project.version} + + + + io.datavines + datavines-connector-api + ${project.version} + + + + \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/src/main/java/io/datavines/metric/expected/plugin/AbstractExpectedValue.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/src/main/java/io/datavines/metric/expected/plugin/AbstractExpectedValue.java new file mode 100644 index 000000000..fe1fa0db1 --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/src/main/java/io/datavines/metric/expected/plugin/AbstractExpectedValue.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.metric.expected.plugin; + +import io.datavines.connector.api.ConnectorFactory; +import io.datavines.metric.api.ExpectedValue; +import io.datavines.spi.PluginLoader; + +import java.util.Map; + +public abstract class AbstractExpectedValue implements ExpectedValue { + + protected ConnectorFactory getConnectorFactory(Map inputParameter) { + String srcConnectorType = inputParameter.get("src_connector_type"); + return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(srcConnectorType); + } + + protected ConnectorFactory getConnectorFactory(String connectorType) { + return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(connectorType); + } +} diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/pom.xml index 89e378587..62bd3305a 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/pom.xml @@ -30,5 +30,12 @@ datavines-metric-expected-daily-avg + + + io.datavines + datavines-metric-expected-base + ${project.version} + + \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/DailyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/DailyAvg.java index 7564f0fd2..5cd4a6e35 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/DailyAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/DailyAvg.java @@ -16,13 +16,13 @@ */ package io.datavines.metric.expected.plugin; -import io.datavines.metric.api.ExpectedValue; +import io.datavines.common.exception.DataVinesException; import java.util.Map; import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; -public class DailyAvg implements ExpectedValue { +public class DailyAvg extends AbstractExpectedValue { @Override public String getName() { @@ -43,10 +43,15 @@ public String getKey(Map inputParameter) { @Override public String getExecuteSql(Map inputParameter) { String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >=date_format(${data_time},'%Y-%m-%d')" + - " and data_time < date_add(date_format(${data_time},'%Y-%m-%d'), interval 1 DAY)" + - " and unique_code = ${unique_code}"; + String engineType = inputParameter.get("engine_type"); + switch (engineType){ + case "spark": + return getConnectorFactory("spark").getMetricScript().dailyAvg(uniqueKey); + case "local": + return getConnectorFactory(inputParameter).getMetricScript().dailyAvg(uniqueKey); + default: + throw new DataVinesException(String.format("engine type %s is not supported", engineType)); + } } @Override diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index af948d983..76e0227c9 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,3 @@ local_daily_avg=io.datavines.metric.expected.plugin.DailyAvg -spark_daily_avg=io.datavines.metric.expected.plugin.SparkDailyAvg -livy_daily_avg=io.datavines.metric.expected.plugin.SparkDailyAvg \ No newline at end of file +spark_daily_avg=io.datavines.metric.expected.plugin.DailyAvg +livy_daily_avg=io.datavines.metric.expected.plugin.DailyAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/pom.xml index 3e7c2c3cb..9dd501dcf 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/pom.xml @@ -30,5 +30,12 @@ datavines-metric-expected-fix + + + io.datavines + datavines-metric-expected-base + ${project.version} + + \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/pom.xml index c1c6603b4..f41f55163 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/pom.xml @@ -29,5 +29,12 @@ datavines-metric-expected-last30day-avg + + + io.datavines + datavines-metric-expected-base + ${project.version} + + \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java index 33ad121bd..bce4640e1 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java @@ -16,13 +16,12 @@ */ package io.datavines.metric.expected.plugin; -import io.datavines.metric.api.ExpectedValue; - +import io.datavines.common.exception.DataVinesException; import java.util.Map; import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; -public class Last30DayAvg implements ExpectedValue { +public class Last30DayAvg extends AbstractExpectedValue { @Override public String getName() { @@ -43,9 +42,15 @@ public String getKey(Map inputParameter) { @Override public String getExecuteSql(Map inputParameter) { String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >= date_sub(date_format(${data_time},'%Y-%m-%d'),interval 30 DAY)" + - " and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY) and unique_code = ${unique_code}"; + String engineType = inputParameter.get("engine_type"); + switch (engineType){ + case "spark": + return getConnectorFactory("spark").getMetricScript().last30DayAvg(uniqueKey); + case "local": + return getConnectorFactory(inputParameter).getMetricScript().last30DayAvg(uniqueKey); + default: + throw new DataVinesException(String.format("engine type %s is not supported", engineType)); + } } @Override diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index 0031556b6..44995d759 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,3 @@ local_last_30d_avg=io.datavines.metric.expected.plugin.Last30DayAvg -spark_last_30d_avg=io.datavines.metric.expected.plugin.SparkLast30DayAvg -livy_last_30d_avg=io.datavines.metric.expected.plugin.SparkLast30DayAvg \ No newline at end of file +spark_last_30d_avg=io.datavines.metric.expected.plugin.Last30DayAvg +livy_last_30d_avg=io.datavines.metric.expected.plugin.Last30DayAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/pom.xml index af3fe8c26..0aadc276b 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/pom.xml @@ -29,5 +29,12 @@ datavines-metric-expected-last7day-avg + + + io.datavines + datavines-metric-expected-base + ${project.version} + + \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/Last7DayAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/Last7DayAvg.java index 0ecce6ad7..58c59208f 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/Last7DayAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/Last7DayAvg.java @@ -16,13 +16,12 @@ */ package io.datavines.metric.expected.plugin; -import io.datavines.metric.api.ExpectedValue; - +import io.datavines.common.exception.DataVinesException; import java.util.Map; import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; -public class Last7DayAvg implements ExpectedValue { +public class Last7DayAvg extends AbstractExpectedValue { @Override public String getName() { @@ -43,9 +42,15 @@ public String getKey(Map inputParameter) { @Override public String getExecuteSql(Map inputParameter) { String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >= date_sub(date_format(${data_time},'%Y-%m-%d'),interval 7 DAY)" + - " and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY) and unique_code = ${unique_code}"; + String engineType = inputParameter.get("engine_type"); + switch (engineType){ + case "spark": + return getConnectorFactory("spark").getMetricScript().last7DayAvg(uniqueKey); + case "local": + return getConnectorFactory(inputParameter).getMetricScript().last7DayAvg(uniqueKey); + default: + throw new DataVinesException(String.format("engine type %s is not supported", engineType)); + } } @Override diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index 391573b28..88c272e68 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,3 @@ local_last_7d_avg=io.datavines.metric.expected.plugin.Last7DayAvg -spark_last_7d_avg=io.datavines.metric.expected.plugin.SparkLast7DayAvg -livy_last_7d_avg=io.datavines.metric.expected.plugin.SparkLast7DayAvg \ No newline at end of file +spark_last_7d_avg=io.datavines.metric.expected.plugin.Last7DayAvg +livy_last_7d_avg=io.datavines.metric.expected.plugin.Last7DayAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/pom.xml index 1ab8db5e3..36b61de5d 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/pom.xml @@ -29,5 +29,12 @@ datavines-metric-expected-monthly-avg + + + io.datavines + datavines-metric-expected-base + ${project.version} + + \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/MonthlyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/MonthlyAvg.java index f0880059f..7324e2ed4 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/MonthlyAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/MonthlyAvg.java @@ -16,13 +16,12 @@ */ package io.datavines.metric.expected.plugin; -import io.datavines.metric.api.ExpectedValue; - +import io.datavines.common.exception.DataVinesException; import java.util.Map; import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; -public class MonthlyAvg implements ExpectedValue { +public class MonthlyAvg extends AbstractExpectedValue { @Override public String getName() { @@ -43,9 +42,15 @@ public String getKey(Map inputParameter) { @Override public String getExecuteSql(Map inputParameter) { String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >= date_format(curdate(), '%Y-%m-01') and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY)" + - " and unique_code = ${unique_code}"; + String engineType = inputParameter.get("engine_type"); + switch (engineType){ + case "spark": + return getConnectorFactory("spark").getMetricScript().monthlyAvg(uniqueKey); + case "local": + return getConnectorFactory(inputParameter).getMetricScript().monthlyAvg(uniqueKey); + default: + throw new DataVinesException(String.format("engine type %s is not supported", engineType)); + } } @Override diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index c969402c4..6fc8acdf6 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,3 @@ local_monthly_avg=io.datavines.metric.expected.plugin.MonthlyAvg -spark_monthly_avg=io.datavines.metric.expected.plugin.SparkMonthlyAvg -livy_monthly_avg=io.datavines.metric.expected.plugin.SparkMonthlyAvg \ No newline at end of file +spark_monthly_avg=io.datavines.metric.expected.plugin.MonthlyAvg +livy_monthly_avg=io.datavines.metric.expected.plugin.MonthlyAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/pom.xml index cd536e85b..daf69d132 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/pom.xml @@ -30,4 +30,12 @@ datavines-metric-expected-none + + + io.datavines + datavines-metric-expected-base + ${project.version} + + + diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/pom.xml index 71401bdd4..7c67a76eb 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/pom.xml @@ -30,4 +30,11 @@ datavines-metric-expected-table-rows + + + io.datavines + datavines-metric-expected-base + ${project.version} + + \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/pom.xml index d71f0cb35..7a3f6316c 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/pom.xml @@ -29,5 +29,12 @@ datavines-metric-expected-target-table-rows + + + io.datavines + datavines-metric-expected-base + ${project.version} + + \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/pom.xml b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/pom.xml index e83f933f4..e02058b21 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/pom.xml @@ -29,5 +29,12 @@ datavines-metric-expected-weekly-avg + + + io.datavines + datavines-metric-expected-base + ${project.version} + + \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java index 08843d370..76891c5a3 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java @@ -16,13 +16,13 @@ */ package io.datavines.metric.expected.plugin; -import io.datavines.metric.api.ExpectedValue; +import io.datavines.common.exception.DataVinesException; import java.util.Map; import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; -public class WeeklyAvg implements ExpectedValue { +public class WeeklyAvg extends AbstractExpectedValue { @Override public String getName() { @@ -43,9 +43,15 @@ public String getZhName() { @Override public String getExecuteSql(Map inputParameter) { String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >= date_sub(${data_time},interval weekday(${data_time}) + 0 day)" + - " and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY) and unique_code = ${unique_code}"; + String engineType = inputParameter.get("engine_type"); + switch (engineType){ + case "spark": + return getConnectorFactory("spark").getMetricScript().weeklyAvg(uniqueKey); + case "local": + return getConnectorFactory(inputParameter).getMetricScript().weeklyAvg(uniqueKey); + default: + throw new DataVinesException(String.format("engine type %s is not supported", engineType)); + } } @Override diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index d6784eaf2..7e8ff7a04 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,3 @@ local_weekly_avg=io.datavines.metric.expected.plugin.WeeklyAvg -spark_weekly_avg=io.datavines.metric.expected.plugin.SparkWeeklyAvg -livy_weekly_avg=io.datavines.metric.expected.plugin.SparkWeeklyAvg \ No newline at end of file +spark_weekly_avg=io.datavines.metric.expected.plugin.WeeklyAvg +livy_weekly_avg=io.datavines.metric.expected.plugin.WeeklyAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/pom.xml b/datavines-metric/datavines-metric-expected-plugins/pom.xml index 60d4db33c..89f07477a 100644 --- a/datavines-metric/datavines-metric-expected-plugins/pom.xml +++ b/datavines-metric/datavines-metric-expected-plugins/pom.xml @@ -41,14 +41,7 @@ datavines-metric-expected-last7day-avg datavines-metric-expected-last30day-avg datavines-metric-expected-none + datavines-metric-expected-base - - - io.datavines - datavines-metric-api - ${project.version} - - - \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-base/src/main/java/io/datavines/metric/plugin/base/BaseSingleTable.java b/datavines-metric/datavines-metric-plugins/datavines-metric-base/src/main/java/io/datavines/metric/plugin/base/BaseSingleTable.java index 519fa71e5..13d959036 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-base/src/main/java/io/datavines/metric/plugin/base/BaseSingleTable.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-base/src/main/java/io/datavines/metric/plugin/base/BaseSingleTable.java @@ -19,9 +19,11 @@ import io.datavines.common.config.CheckResult; import io.datavines.common.config.ConfigChecker; import io.datavines.common.entity.ExecuteSql; +import io.datavines.connector.api.ConnectorFactory; import io.datavines.metric.api.ConfigItem; import io.datavines.metric.api.MetricLevel; import io.datavines.metric.api.SqlMetric; +import io.datavines.spi.PluginLoader; import org.apache.commons.lang3.StringUtils; import java.util.*; @@ -90,4 +92,9 @@ private void addFiltersIntoInvalidateItemsSql() { public MetricLevel getLevel() { return MetricLevel.TABLE; } + + protected ConnectorFactory getConnectorFactory(Map inputParameter) { + String srcConnectorType = inputParameter.get("src_connector_type"); + return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(srcConnectorType); + } } \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/java/io/datavines/metric/plugin/ColumnAvgLength.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/java/io/datavines/metric/plugin/ColumnAvgLength.java index 3238f7d27..1d26ab6fc 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/java/io/datavines/metric/plugin/ColumnAvgLength.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/java/io/datavines/metric/plugin/ColumnAvgLength.java @@ -68,9 +68,9 @@ public ExecuteSql getInvalidateItems(Map inputParameter) { public ExecuteSql getActualValue(Map inputParameter) { String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); ExecuteSql executeSql = new ExecuteSql(); - executeSql.setResultTable("invalidate_count_"+uniqueKey); + executeSql.setResultTable("invalidate_count_" + uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select avg(${length_key}) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().avgLengthActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/java/io/datavines/metric/plugin/ColumnAvg.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/java/io/datavines/metric/plugin/ColumnAvg.java index 09ae839bc..06b21303b 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/java/io/datavines/metric/plugin/ColumnAvg.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/java/io/datavines/metric/plugin/ColumnAvg.java @@ -21,6 +21,8 @@ import io.datavines.metric.api.MetricDimension; import io.datavines.metric.api.MetricType; import io.datavines.metric.plugin.base.BaseSingleTableColumn; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.List; @@ -30,6 +32,8 @@ public class ColumnAvg extends BaseSingleTableColumn { + private static final Logger log = LoggerFactory.getLogger(ColumnAvg.class); + public ColumnAvg(){ super(); } @@ -70,7 +74,7 @@ public ExecuteSql getActualValue(Map inputParameter) { ExecuteSql executeSql = new ExecuteSql(); executeSql.setResultTable("invalidate_count_"+uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select avg(${column}) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().avgActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java index 1eebe5323..12768db55 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java @@ -60,7 +60,7 @@ public boolean isInvalidateItemsCanOutput() { @Override public void prepare(Map config) { if (config.containsKey("column")) { - filters.add("(${column} is null or ${column} = '')"); + filters.add(getConnectorFactory(config).getMetricScript().columnIsBlank()); } super.prepare(config); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/java/io/datavines/metric/plugin/ColumnDistinct.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/java/io/datavines/metric/plugin/ColumnDistinct.java index 5caf6585d..d86a49104 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/java/io/datavines/metric/plugin/ColumnDistinct.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/java/io/datavines/metric/plugin/ColumnDistinct.java @@ -65,7 +65,7 @@ public ExecuteSql getActualValue(Map inputParameter) { ExecuteSql executeSql = new ExecuteSql(); executeSql.setResultTable("invalidate_count_" + uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select count(distinct(${column})) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().countDistinctActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java index b73dd30b8..812abe3a8 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java @@ -67,7 +67,7 @@ public void prepare(Map config) { } if (config.containsKey("column")) { - invalidateItemsSql.append(" group by ${column} having count(${column}) > 1"); + invalidateItemsSql.append(getConnectorFactory(config).getMetricScript().groupByHavingCountForDuplicate()); } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java index a7a4caada..179384325 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java @@ -18,16 +18,17 @@ import io.datavines.common.entity.ExecuteSql; import io.datavines.common.enums.DataVinesDataType; +import io.datavines.connector.api.ConnectorFactory; import io.datavines.metric.api.MetricActualValueType; import io.datavines.metric.api.MetricDimension; import io.datavines.metric.api.MetricType; import io.datavines.metric.plugin.base.BaseSingleTableColumn; +import io.datavines.spi.PluginLoader; import java.util.Arrays; import java.util.List; import java.util.Map; -import static io.datavines.common.CommonConstants.SPARK; import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; public class ColumnHistogram extends BaseSingleTableColumn { @@ -70,41 +71,26 @@ public ExecuteSql getInvalidateItems(Map inputParameter) { public ExecuteSql getActualValue(Map inputParameter) { String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); ExecuteSql executeSql = new ExecuteSql(); - executeSql.setResultTable("invalidate_count_"+uniqueKey); + executeSql.setResultTable("invalidate_count_" + uniqueKey); StringBuilder actualValueSql = new StringBuilder(); String engineType = inputParameter.get("engine_type"); - if (SPARK.equalsIgnoreCase(engineType)) { - actualValueSql.append("select concat(k, '\001', cast(count as ${string_type})) as actual_value_").append(uniqueKey).append(" from (select if(${column} is null, 'NULL', cast(${column} as ${string_type})) as k, count(1) as count from ${table}"); - if (!filters.isEmpty()) { - actualValueSql.append(" where ").append(String.join(" and ", filters)); - } - - actualValueSql.append(" group by ${column} order by count desc limit 50) T"); - } else { - String srcConnectorType = inputParameter.get("src_connector_type"); - switch (srcConnectorType){ - case "oracle": - actualValueSql.append("select concat(concat(k, '\001'), count) as actual_value_").append(uniqueKey).append(" from (select ${if_case_key} as k, count(1) as count from ${table}"); - break; - default: - actualValueSql.append("select concat(k, '\001', cast(count as ${string_type})) as actual_value_").append(uniqueKey).append(" from (select ${if_case_key} as k, count(1) as count from ${table}"); - break; - } - - if (!filters.isEmpty()) { - actualValueSql.append(" where ").append(String.join(" and ", filters)); - } - actualValueSql.append(" group by ${column} order by count desc "); - switch (srcConnectorType){ - case "oracle": - // oracle the rownum filter should be placed at the outermost layer of the query, otherwise it cannot be filtered - actualValueSql.append(" ) T where ${limit_top_50_key} "); - break; - default: - actualValueSql.append(" ${limit_top_50_key}) T"); - break; - } + String where = ""; + switch (engineType){ + case "spark": + if (!filters.isEmpty()) { + where = " where " + String.join(" and ", filters); + } + actualValueSql.append(getConnectorFactory().getMetricScript().histogramActualValue(uniqueKey,where)); + break; + case "local": + if (!filters.isEmpty()) { + where = " where " + String.join(" and ", filters); + } + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().histogramActualValue(uniqueKey,where)); + break; + default: + break; } executeSql.setSql(actualValueSql.toString()); @@ -121,4 +107,8 @@ public String getActualValueType() { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + private ConnectorFactory getConnectorFactory() { + return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin("spark"); + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/java/io/datavines/metric/plugin/ColumnInEnums.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/java/io/datavines/metric/plugin/ColumnInEnums.java index f0b0e3b40..14b31691d 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/java/io/datavines/metric/plugin/ColumnInEnums.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/java/io/datavines/metric/plugin/ColumnInEnums.java @@ -62,7 +62,7 @@ public boolean isInvalidateItemsCanOutput() { @Override public void prepare(Map config) { if (config.containsKey("enum_list") && config.containsKey("column")) { - filters.add(" (${column} in ( ${enum_list} )) "); + filters.add(getConnectorFactory(config).getMetricScript().columnInEnums()); } super.prepare(config); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/java/io/datavines/metric/plugin/ColumnLength.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/java/io/datavines/metric/plugin/ColumnLength.java index 2b3b3a28d..a09a6c66d 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/java/io/datavines/metric/plugin/ColumnLength.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/java/io/datavines/metric/plugin/ColumnLength.java @@ -67,7 +67,7 @@ public boolean isInvalidateItemsCanOutput() { public void prepare(Map config) { if (config.containsKey("length") && config.containsKey("comparator") && config.containsKey("column")) { - filters.add(" ${length_key} ${comparator} ${length}"); + filters.add(getConnectorFactory(config).getMetricScript().columnLengthCompare()); } super.prepare(config); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java index 50773e553..7742c56ef 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java @@ -66,7 +66,7 @@ public boolean isInvalidateItemsCanOutput() { public void prepare(Map config) { if (config.containsKey("table") && config.containsKey("column") && config.containsKey("regexp") ) { - filters.add(" ${not_regex_key} "); + filters.add(getConnectorFactory(config).getMetricScript().columnNotMatchRegex()); } super.prepare(config); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java index 75e73223f..584248907 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java @@ -64,7 +64,7 @@ public boolean isInvalidateItemsCanOutput() { public void prepare(Map config) { if (config.containsKey("table") && config.containsKey("column") && config.containsKey("regexp") ) { - filters.add(" ${regex_key} "); + filters.add(getConnectorFactory(config).getMetricScript().columnMatchRegex()); } super.prepare(config); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/java/io/datavines/metric/plugin/ColumnMaxLength.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/java/io/datavines/metric/plugin/ColumnMaxLength.java index 68b09b1e5..7f62c6cee 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/java/io/datavines/metric/plugin/ColumnMaxLength.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/java/io/datavines/metric/plugin/ColumnMaxLength.java @@ -70,7 +70,7 @@ public ExecuteSql getActualValue(Map inputParameter) { ExecuteSql executeSql = new ExecuteSql(); executeSql.setResultTable("invalidate_count_"+uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select max(${length_key}) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().maxLengthActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/java/io/datavines/metric/plugin/ColumnMax.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/java/io/datavines/metric/plugin/ColumnMax.java index 1b7fd22fb..02c67d544 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/java/io/datavines/metric/plugin/ColumnMax.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/java/io/datavines/metric/plugin/ColumnMax.java @@ -68,9 +68,9 @@ public ExecuteSql getInvalidateItems(Map inputParameter) { public ExecuteSql getActualValue(Map inputParameter) { String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); ExecuteSql executeSql = new ExecuteSql(); - executeSql.setResultTable("invalidate_count_"+uniqueKey); + executeSql.setResultTable("invalidate_count_" + uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select max(${column}) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().maxActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/java/io/datavines/metric/plugin/ColumnMinLength.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/java/io/datavines/metric/plugin/ColumnMinLength.java index 8dd810d7a..99a0260f4 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/java/io/datavines/metric/plugin/ColumnMinLength.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/java/io/datavines/metric/plugin/ColumnMinLength.java @@ -73,7 +73,7 @@ public ExecuteSql getActualValue(Map inputParameter) { ExecuteSql executeSql = new ExecuteSql(); executeSql.setResultTable("invalidate_count_" + uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select min(${length_key}) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().minLengthActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/java/io/datavines/metric/plugin/ColumnMin.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/java/io/datavines/metric/plugin/ColumnMin.java index 91b8ff449..4cc47d171 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/java/io/datavines/metric/plugin/ColumnMin.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/java/io/datavines/metric/plugin/ColumnMin.java @@ -70,7 +70,7 @@ public ExecuteSql getActualValue(Map inputParameter) { ExecuteSql executeSql = new ExecuteSql(); executeSql.setResultTable("invalidate_count_" + uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select min(${column}) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().minActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java index 78f7423ed..bdd1168dd 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java @@ -64,7 +64,7 @@ public boolean isInvalidateItemsCanOutput() { @Override public void prepare(Map config) { if (config.containsKey("enum_list") && config.containsKey("column")) { - filters.add(" (${column} not in ( ${enum_list} ) or ${column} is null) "); + filters.add(getConnectorFactory(config).getMetricScript().columnNotInEnums()); } super.prepare(config); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java index 96fc100bd..897e0a778 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java @@ -59,7 +59,7 @@ public boolean isInvalidateItemsCanOutput() { @Override public void prepare(Map config) { if (config.containsKey("column")) { - filters.add(" ${column} is not null "); + filters.add(getConnectorFactory(config).getMetricScript().columnNotNull()); } super.prepare(config); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java index d5e8881a9..a8ea50dcf 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java @@ -60,7 +60,7 @@ public boolean isInvalidateItemsCanOutput() { @Override public void prepare(Map config) { if (config.containsKey("column")) { - filters.add("(${column} is null )"); + filters.add(getConnectorFactory(config).getMetricScript().columnIsNull()); } super.prepare(config); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/java/io/datavines/metric/plugin/ColumnStdDev.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/java/io/datavines/metric/plugin/ColumnStdDev.java index e1ca40284..66e52f6e0 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/java/io/datavines/metric/plugin/ColumnStdDev.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/java/io/datavines/metric/plugin/ColumnStdDev.java @@ -70,7 +70,7 @@ public ExecuteSql getActualValue(Map inputParameter) { ExecuteSql executeSql = new ExecuteSql(); executeSql.setResultTable("invalidate_count_" + uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select ${std_dev_key}(${column}) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().stdDevActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/java/io/datavines/metric/plugin/ColumnSum.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/java/io/datavines/metric/plugin/ColumnSum.java index 559a176b9..0c7f47337 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/java/io/datavines/metric/plugin/ColumnSum.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/java/io/datavines/metric/plugin/ColumnSum.java @@ -70,7 +70,7 @@ public ExecuteSql getActualValue(Map inputParameter) { ExecuteSql executeSql = new ExecuteSql(); executeSql.setResultTable("invalidate_count_" + uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select sum(${column}) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().sumActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/java/io/datavines/metric/plugin/ColumnUnique.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/java/io/datavines/metric/plugin/ColumnUnique.java index 2aede8bfb..3582b1004 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/java/io/datavines/metric/plugin/ColumnUnique.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/java/io/datavines/metric/plugin/ColumnUnique.java @@ -66,7 +66,7 @@ public void prepare(Map config) { } if (config.containsKey("column")) { - invalidateItemsSql.append(" group by ${column} having count(1) = 1"); + invalidateItemsSql.append(getConnectorFactory(config).getMetricScript().groupByHavingCountForUnique()); } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/java/io/datavines/metric/plugin/ColumnValueBetween.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/java/io/datavines/metric/plugin/ColumnValueBetween.java index 8f5811869..4c4ed85cd 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/java/io/datavines/metric/plugin/ColumnValueBetween.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/java/io/datavines/metric/plugin/ColumnValueBetween.java @@ -65,11 +65,11 @@ public boolean isInvalidateItemsCanOutput() { public void prepare(Map config) { if (config.containsKey("min")) { - filters.add("${column} >= ${min}"); + filters.add(getConnectorFactory(config).getMetricScript().columnGteMin()); } if (config.containsKey("max")) { - filters.add("${column} <= ${max}"); + filters.add(getConnectorFactory(config).getMetricScript().columnLteMax()); } super.prepare(config); diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/java/io/datavines/metric/plugin/ColumnVariance.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/java/io/datavines/metric/plugin/ColumnVariance.java index 6d38b0f6f..b22a02612 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/java/io/datavines/metric/plugin/ColumnVariance.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/java/io/datavines/metric/plugin/ColumnVariance.java @@ -70,7 +70,7 @@ public ExecuteSql getActualValue(Map inputParameter) { ExecuteSql executeSql = new ExecuteSql(); executeSql.setResultTable("invalidate_count_" + uniqueKey); StringBuilder actualValueSql = new StringBuilder(); - actualValueSql.append("select ${variance_key}(${column}) as actual_value_").append(uniqueKey).append(" from ${table}"); + actualValueSql.append(getConnectorFactory(inputParameter).getMetricScript().varianceActualValue(uniqueKey)); if (!filters.isEmpty()) { actualValueSql.append(" where ").append(String.join(" and ", filters)); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/java/io/datavines/metric/plugin/Freshness.java b/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/java/io/datavines/metric/plugin/Freshness.java index c8ea33e63..a0e1ff285 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/java/io/datavines/metric/plugin/Freshness.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/java/io/datavines/metric/plugin/Freshness.java @@ -22,7 +22,7 @@ import io.datavines.metric.api.MetricType; import io.datavines.metric.plugin.base.BaseSingleTableColumnNotUseView; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -67,13 +67,13 @@ public boolean isInvalidateItemsCanOutput() { public void prepare(Map config) { if (config.containsKey("column") && config.containsKey("datetime_format") && config.containsKey("deadline_time") && config.containsKey("begin_time")) { - filters.add(" (DATE_FORMAT(${column}, '${datetime_format}') <= DATE_FORMAT('${deadline_time}', '${datetime_format}') ) AND (DATE_FORMAT(${column}, '${datetime_format}') >= DATE_FORMAT('${begin_time}', '${datetime_format}')) "); + filters.add(getConnectorFactory(config).getMetricScript().timeBetweenWithFormat()); } super.prepare(config); } @Override public List suitableType() { - return Collections.singletonList(DataVinesDataType.DATE_TIME_TYPE); + return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } } diff --git a/datavines-metric/datavines-metric-plugins/pom.xml b/datavines-metric/datavines-metric-plugins/pom.xml index 4895cb899..fa98824fb 100644 --- a/datavines-metric/datavines-metric-plugins/pom.xml +++ b/datavines-metric/datavines-metric-plugins/pom.xml @@ -67,6 +67,12 @@ datavines-metric-api ${project.version} + + + io.datavines + datavines-connector-api + ${project.version} + diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/DataSourceController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/DataSourceController.java index 7dbe01715..c45a820b2 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/controller/DataSourceController.java +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/DataSourceController.java @@ -149,7 +149,8 @@ public Object getConnectorTypeList() { List items = new ArrayList<>(); connectorList.forEach(it -> { - if (!"file".equalsIgnoreCase(it)) { + ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(it); + if (connectorFactory.showInFrontend()) { Item item = new Item(it,it); items.add(item); } diff --git a/datavines-spi/src/main/java/io/datavines/spi/PluginLoader.java b/datavines-spi/src/main/java/io/datavines/spi/PluginLoader.java index a1f7aa934..da57ecad1 100644 --- a/datavines-spi/src/main/java/io/datavines/spi/PluginLoader.java +++ b/datavines-spi/src/main/java/io/datavines/spi/PluginLoader.java @@ -378,7 +378,7 @@ private void loadResource(Map> pluginClasses, ClassLoader class line = line.substring(0, ci); } line = line.trim(); - if (line.length() > 0) { + if (!line.isEmpty()) { try { String name = null; int i = line.indexOf('='); @@ -386,7 +386,7 @@ private void loadResource(Map> pluginClasses, ClassLoader class name = line.substring(0, i).trim(); line = line.substring(i + 1).trim(); } - if (line.length() > 0 ) { + if (!line.isEmpty()) { loadClass(pluginClasses, resourceUrl, Class.forName(line, true, classLoader), name); } } catch (Throwable t) { @@ -412,7 +412,7 @@ private void loadClass(Map> pluginClasses, if (StringUtils.isEmpty(name)) { name = findAnnotationName(clazz); - if (name.length() == 0) { + if (name.isEmpty()) { throw new IllegalStateException("No such plugin name for the class " + clazz.getName() + " in the config " + resourceUrl); } }