From 45b848b5698cb89a837e726e237aafb027ceae57 Mon Sep 17 00:00:00 2001 From: tangjiafu Date: Sun, 14 Aug 2022 15:33:23 +0800 Subject: [PATCH 01/23] db2 type --- .../connector-jdbc/pom.xml | 6 +- .../jdbc/internal/dialect/db2/DB2Dialect.java | 40 +++++++ .../dialect/db2/DB2DialectFactory.java | 42 +++++++ .../dialect/db2/DB2JdbcRowConverter.java | 37 +++++++ .../internal/dialect/db2/DB2TypeMapper.java | 104 ++++++++++++++++++ 5 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java diff --git a/seatunnel-connectors-v2/connector-jdbc/pom.xml b/seatunnel-connectors-v2/connector-jdbc/pom.xml index 7e49bcce0ce..9721870da9d 100644 --- a/seatunnel-connectors-v2/connector-jdbc/pom.xml +++ b/seatunnel-connectors-v2/connector-jdbc/pom.xml @@ -108,6 +108,10 @@ com.oracle.database.jdbc ojdbc8 + + com.ibm.db2 + jcc + - + \ No newline at end of file diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java new file mode 100644 index 00000000000..adb09419de2 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2; + +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.JdbcRowConverter; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; + +public class DB2Dialect implements JdbcDialect { + + @Override + public String dialectName() { + return "DB2"; + } + + @Override + public JdbcRowConverter getRowConverter() { + return new DB2JdbcRowConverter(); + } + + @Override + public JdbcDialectTypeMapper getJdbcDialectTypeMapper() { + return new DB2TypeMapper(); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java new file mode 100644 index 00000000000..acc467f6401 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2; + + +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; + +import com.google.auto.service.AutoService; + +/** + * Factory for {@link DB2Dialect}. + */ + +@AutoService(JdbcDialectFactory.class) +public class DB2DialectFactory implements JdbcDialectFactory { + + @Override + public boolean acceptsURL(String url) { + return url.startsWith("jdbc:mysql:"); + } + + @Override + public JdbcDialect create() { + return new DB2Dialect(); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java new file mode 100644 index 00000000000..80076c193d7 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.AbstractJdbcRowConverter; + +public class DB2JdbcRowConverter extends AbstractJdbcRowConverter { + @Override + public String converterName() { + return "DB2"; + } + + @Override + public SeaTunnelRow toInternal(ResultSet rs, ResultSetMetaData metaData, SeaTunnelRowType typeInfo) throws SQLException { + return super.toInternal(rs, metaData, typeInfo); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java new file mode 100644 index 00000000000..a8c84eb2cf3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2; + + +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +public class DB2TypeMapper implements JdbcDialectTypeMapper { + + private static final Logger LOG = LoggerFactory.getLogger(JdbcDialect.class); + + // reference https://www.ibm.com/docs/en/ssw_ibm_i_75/pdf/rbafzpdf.pdf + + // ============================data types===================== + private static final String DB2_BOOLEAN = "BOOLEAN"; + private static final String DB2_ROWID = "ROWID"; + private static final String DB2_SMALLINT = "SMALLINT"; + private static final String DB2_INTEGER = "INTEGER"; + private static final String DB2_INT = "INT"; + private static final String DB2_BIGINT = "BIGINT"; + // exact + private static final String DB2_DECIMAL = "DECIMAL"; + private static final String DB2_DEC = "DEC"; + private static final String DB2_NUMERIC = "NUMERIC"; + private static final String DB2_NUM = "NUM"; + + + // float + private static final String DB2_REAL = "REAL"; + private static final String DB2_FLOAT = "FLOAT"; + private static final String DB2_DOUBLE = "DOUBLE"; + private static final String DB2_DOUBLE_PRECISION = "DOUBLE PRECISION"; + private static final String DB2_DECFLOAT = "DECFLOAT"; + private static final String DB2_DECDOUBLE = "DECDOUBLE"; + + // string + private static final String DB2_CHAR = "CHAR"; + private static final String DB2_VARCHAR = "VARCHAR"; + private static final String DB2_CLOB = "CLOB"; + private static final String DB2_LONGVARCHAR = "LONG VARCHAR"; + // graphic + private static final String DB2_GRAPHIC = "GRAPHIC"; + private static final String DB2_VARGRAPHIC = "VARGRAPHIC"; + private static final String DB2_LONG_VARGRAPHIC = "LONG VARGRAPHIC"; + private static final String DB2_DBCLOB = "DBCLOB"; + + // ---------------------------binary--------------------------- + private static final String DB2_BINARY = "BINARY"; + private static final String DB2_VARBINARY = "VARBINARY"; + + // ------------------------------time------------------------- + private static final String DB2_DATE = "DATE"; + private static final String DB2_TIME = "TIME"; + private static final String DB2_TIMESTAMP = "TIMESTAMP"; + + + // ------------------------------blob------------------------- + private static final String DB2_BLOB = "BLOB"; + private static final String DB2_NCHAR_MAPPING = "NCHAR_MAPPING"; + + // other + private static final String DB2_XML = "XML"; + private static final String DB2_LOB = "XML"; + private static final String DB2_DATALINK = "DATALINK"; + + + @SuppressWarnings("checkstyle:MagicNumber") + @Override + public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) throws SQLException { + String columnType = metadata.getColumnTypeName(colIndex).toUpperCase(); + int precision = metadata.getPrecision(colIndex); + int scale = metadata.getScale(colIndex); + switch (columnType) { + case DB2_BOOLEAN: + + default: + final String jdbcColumnName = metadata.getColumnName(colIndex); + throw new UnsupportedOperationException( + String.format("Doesn't support DM2 type '%s' on column '%s' yet.", columnType, jdbcColumnName)); + } + } +} From fc7abf90c10c74b378daebfe3648ead4ac84d416 Mon Sep 17 00:00:00 2001 From: tangjiafu Date: Sun, 14 Aug 2022 17:03:17 +0800 Subject: [PATCH 02/23] update db2 mapper #2367 --- .../internal/dialect/db2/DB2TypeMapper.java | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java index a8c84eb2cf3..959425612e5 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java @@ -18,6 +18,10 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; @@ -32,9 +36,9 @@ public class DB2TypeMapper implements JdbcDialectTypeMapper { private static final Logger LOG = LoggerFactory.getLogger(JdbcDialect.class); // reference https://www.ibm.com/docs/en/ssw_ibm_i_75/pdf/rbafzpdf.pdf - // ============================data types===================== private static final String DB2_BOOLEAN = "BOOLEAN"; + private static final String DB2_ROWID = "ROWID"; private static final String DB2_SMALLINT = "SMALLINT"; private static final String DB2_INTEGER = "INTEGER"; @@ -59,7 +63,7 @@ public class DB2TypeMapper implements JdbcDialectTypeMapper { private static final String DB2_CHAR = "CHAR"; private static final String DB2_VARCHAR = "VARCHAR"; private static final String DB2_CLOB = "CLOB"; - private static final String DB2_LONGVARCHAR = "LONG VARCHAR"; + private static final String DB2_LONG_VARCHAR = "LONG VARCHAR"; // graphic private static final String DB2_GRAPHIC = "GRAPHIC"; private static final String DB2_VARGRAPHIC = "VARGRAPHIC"; @@ -82,7 +86,7 @@ public class DB2TypeMapper implements JdbcDialectTypeMapper { // other private static final String DB2_XML = "XML"; - private static final String DB2_LOB = "XML"; + private static final String DB2_LOB = "LOB"; private static final String DB2_DATALINK = "DATALINK"; @@ -91,10 +95,57 @@ public class DB2TypeMapper implements JdbcDialectTypeMapper { public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) throws SQLException { String columnType = metadata.getColumnTypeName(colIndex).toUpperCase(); int precision = metadata.getPrecision(colIndex); - int scale = metadata.getScale(colIndex); switch (columnType) { case DB2_BOOLEAN: - + return BasicType.BOOLEAN_TYPE; + case DB2_SMALLINT: + return BasicType.SHORT_TYPE; + case DB2_INT: + case DB2_INTEGER: + return BasicType.INT_TYPE; + case DB2_BIGINT: + return BasicType.LONG_TYPE; + case DB2_DECIMAL: + case DB2_DEC: + case DB2_NUMERIC: + case DB2_NUM: + if (precision > 0) { + return new DecimalType(precision, metadata.getScale(colIndex)); + } + return new DecimalType(38, 18); + case DB2_REAL: + return BasicType.FLOAT_TYPE; + case DB2_FLOAT: + case DB2_DOUBLE: + case DB2_DOUBLE_PRECISION: + case DB2_DECFLOAT: + case DB2_DECDOUBLE: + return BasicType.DOUBLE_TYPE; + case DB2_CHAR: + case DB2_VARCHAR: + case DB2_CLOB: + case DB2_LONG_VARCHAR: + case DB2_GRAPHIC: + case DB2_VARGRAPHIC: + case DB2_LONG_VARGRAPHIC: + case DB2_DBCLOB: + return BasicType.STRING_TYPE; + case DB2_BINARY: + case DB2_VARBINARY: + case DB2_BLOB: + case DB2_NCHAR_MAPPING: + return PrimitiveByteArrayType.INSTANCE; + case DB2_DATE: + return LocalTimeType.LOCAL_DATE_TYPE; + case DB2_TIME: + return LocalTimeType.LOCAL_TIME_TYPE; + case DB2_TIMESTAMP: + return LocalTimeType.LOCAL_DATE_TIME_TYPE; + case DB2_ROWID: + // should support + case DB2_XML: + case DB2_LOB: + case DB2_DATALINK: default: final String jdbcColumnName = metadata.getColumnName(colIndex); throw new UnsupportedOperationException( From f412b09cb2418122a5a600d2efc9b1c91423cecf Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Mon, 22 Aug 2022 23:09:38 +0800 Subject: [PATCH 03/23] fix url --- .../seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java | 2 +- .../seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java index acc467f6401..6f629976c3e 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java @@ -32,7 +32,7 @@ public class DB2DialectFactory implements JdbcDialectFactory { @Override public boolean acceptsURL(String url) { - return url.startsWith("jdbc:mysql:"); + return url.startsWith("jdbc:db2:"); } @Override diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java index 959425612e5..6745809e31d 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java @@ -49,8 +49,6 @@ public class DB2TypeMapper implements JdbcDialectTypeMapper { private static final String DB2_DEC = "DEC"; private static final String DB2_NUMERIC = "NUMERIC"; private static final String DB2_NUM = "NUM"; - - // float private static final String DB2_REAL = "REAL"; private static final String DB2_FLOAT = "FLOAT"; @@ -58,7 +56,6 @@ public class DB2TypeMapper implements JdbcDialectTypeMapper { private static final String DB2_DOUBLE_PRECISION = "DOUBLE PRECISION"; private static final String DB2_DECFLOAT = "DECFLOAT"; private static final String DB2_DECDOUBLE = "DECDOUBLE"; - // string private static final String DB2_CHAR = "CHAR"; private static final String DB2_VARCHAR = "VARCHAR"; From b77e8a03c99eea2f3218c8a3d4b8cac1b669be46 Mon Sep 17 00:00:00 2001 From: laglangyue Date: Sat, 3 Sep 2022 19:14:20 +0800 Subject: [PATCH 04/23] add db2 e2e --- docs/en/connector-v2/sink/Jdbc.md | 22 +-- docs/en/connector-v2/source/Jdbc.md | 1 + .../connector-jdbc/pom.xml | 6 +- .../jdbc/internal/dialect/db2/DB2Dialect.java | 24 +-- .../dialect/db2/DB2DialectFactory.java | 1 - .../dialect/db2/DB2JdbcRowConverter.java | 2 + .../internal/dialect/db2/DB2TypeMapper.java | 18 +-- .../e2e/flink/v2/jdbc/JdbcDb2IT.java | 142 +++++++++++++++++ .../resources/jdbc/init_sql/db2_init.conf | 108 +++++++++++++ .../jdbc/jdbc_db2_source_and_sink.conf | 89 +++++++++++ .../connector-jdbc-spark-e2e/pom.xml | 13 ++ .../e2e/spark/v2/jdbc/JdbcDb2IT.java | 150 ++++++++++++++++++ .../resources/jdbc/init_sql/db2_init.conf | 109 +++++++++++++ .../jdbc/jdbc_db2_source_and_sink.conf | 90 +++++++++++ .../pom.xml | 7 +- .../examples/jdbc_db2_source_and_sink.conf | 90 +++++++++++ 16 files changed, 833 insertions(+), 39 deletions(-) create mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/jdbc/JdbcDb2IT.java create mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/init_sql/db2_init.conf create mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf create mode 100644 seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java create mode 100644 seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/init_sql/db2_init.conf create mode 100644 seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf create mode 100644 seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/jdbc_db2_source_and_sink.conf diff --git a/docs/en/connector-v2/sink/Jdbc.md b/docs/en/connector-v2/sink/Jdbc.md index 4114b1ca1b9..9f4bfc6d8fd 100644 --- a/docs/en/connector-v2/sink/Jdbc.md +++ b/docs/en/connector-v2/sink/Jdbc.md @@ -108,16 +108,17 @@ In the case of is_exactly_once = "true", Xa transactions are used. This requires there are some reference value for params above. -| datasource | driver | url | xa_data_source_class_name | maven | -|------------| -------------------------------------------- | ------------------------------------------------------------ | -------------------------------------------------- | ------------------------------------------------------------ | -| MySQL | com.mysql.cj.jdbc.Driver | jdbc:mysql://localhost:3306/test | com.mysql.cj.jdbc.MysqlXADataSource | https://mvnrepository.com/artifact/mysql/mysql-connector-java | -| PostgreSQL | org.postgresql.Driver | jdbc:postgresql://localhost:5432/postgres | org.postgresql.xa.PGXADataSource | https://mvnrepository.com/artifact/org.postgresql/postgresql | -| DM | dm.jdbc.driver.DmDriver | jdbc:dm://localhost:5236 | dm.jdbc.driver.DmdbXADataSource | https://mvnrepository.com/artifact/com.dameng/DmJdbcDriver18 | -| Phoenix | org.apache.phoenix.queryserver.client.Driver | jdbc:phoenix:thin:url=http://localhost:8765;serialization=PROTOBUF | / | https://mvnrepository.com/artifact/com.aliyun.phoenix/ali-phoenix-shaded-thin-client | -| SQL Server | com.microsoft.sqlserver.jdbc.SQLServerDriver | jdbc:microsoft:sqlserver://localhost:1433 | com.microsoft.sqlserver.jdbc.SQLServerXADataSource | https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc | -| Oracle | oracle.jdbc.OracleDriver | jdbc:oracle:thin:@localhost:1521/xepdb1 | oracle.jdbc.xa.OracleXADataSource | https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8 | -| GBase8a | com.gbase.jdbc.Driver | jdbc:gbase://e2e_gbase8aDb:5258/test | / | https://www.gbase8.cn/wp-content/uploads/2020/10/gbase-connector-java-8.3.81.53-build55.5.7-bin_min_mix.jar | -| StarRocks | com.mysql.cj.jdbc.Driver | jdbc:mysql://localhost:3306/test | / | https://mvnrepository.com/artifact/mysql/mysql-connector-java | +| datasource | driver | url | xa_data_source_class_name | maven | +|------------|----------------------------------------------|--------------------------------------------------------------------|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------| +| MySQL | com.mysql.cj.jdbc.Driver | jdbc:mysql://localhost:3306/test | com.mysql.cj.jdbc.MysqlXADataSource | https://mvnrepository.com/artifact/mysql/mysql-connector-java | +| PostgreSQL | org.postgresql.Driver | jdbc:postgresql://localhost:5432/postgres | org.postgresql.xa.PGXADataSource | https://mvnrepository.com/artifact/org.postgresql/postgresql | +| DM | dm.jdbc.driver.DmDriver | jdbc:dm://localhost:5236 | dm.jdbc.driver.DmdbXADataSource | https://mvnrepository.com/artifact/com.dameng/DmJdbcDriver18 | +| Phoenix | org.apache.phoenix.queryserver.client.Driver | jdbc:phoenix:thin:url=http://localhost:8765;serialization=PROTOBUF | / | https://mvnrepository.com/artifact/com.aliyun.phoenix/ali-phoenix-shaded-thin-client | +| SQL Server | com.microsoft.sqlserver.jdbc.SQLServerDriver | jdbc:microsoft:sqlserver://localhost:1433 | com.microsoft.sqlserver.jdbc.SQLServerXADataSource | https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc | +| Oracle | oracle.jdbc.OracleDriver | jdbc:oracle:thin:@localhost:1521/xepdb1 | oracle.jdbc.xa.OracleXADataSource | https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8 | +| GBase8a | com.gbase.jdbc.Driver | jdbc:gbase://e2e_gbase8aDb:5258/test | / | https://www.gbase8.cn/wp-content/uploads/2020/10/gbase-connector-java-8.3.81.53-build55.5.7-bin_min_mix.jar | +| StarRocks | com.mysql.cj.jdbc.Driver | jdbc:mysql://localhost:3306/test | / | https://mvnrepository.com/artifact/mysql/mysql-connector-java | +| db2 | com.ibm.db2.jcc.DB2Driver | jdbc:db2://localhost:50000/testdb | com.ibm.db2.jcc.DB2XADataSource | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc/db2jcc4 | ## Example @@ -166,3 +167,4 @@ jdbc { - [Feature] Support SQL Server JDBC Source ([2646](https://github.com/apache/incubator-seatunnel/pull/2646)) - [Feature] Support Oracle JDBC Source ([2550](https://github.com/apache/incubator-seatunnel/pull/2550)) - [Feature] Support StarRocks JDBC Source ([3060](https://github.com/apache/incubator-seatunnel/pull/3060)) +- [Feature] Support DB2 JDBC Source ([2410](https://github.com/apache/incubator-seatunnel/pull/2410)) \ No newline at end of file diff --git a/docs/en/connector-v2/source/Jdbc.md b/docs/en/connector-v2/source/Jdbc.md index 79a45048f86..0a6293b82bf 100644 --- a/docs/en/connector-v2/source/Jdbc.md +++ b/docs/en/connector-v2/source/Jdbc.md @@ -100,6 +100,7 @@ there are some reference value for params above. | oracle | oracle.jdbc.OracleDriver | jdbc:oracle:thin:@localhost:1521/xepdb1 | https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8 | | gbase8a | com.gbase.jdbc.Driver | jdbc:gbase://e2e_gbase8aDb:5258/test | https://www.gbase8.cn/wp-content/uploads/2020/10/gbase-connector-java-8.3.81.53-build55.5.7-bin_min_mix.jar | | starrocks | com.mysql.cj.jdbc.Driver | jdbc:mysql://localhost:3306/test | https://mvnrepository.com/artifact/mysql/mysql-connector-java | +| db2 | com.ibm.db2.jcc.DB2Driver | jdbc:db2://localhost:50000/testdb | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc/db2jcc4 | ## Example diff --git a/seatunnel-connectors-v2/connector-jdbc/pom.xml b/seatunnel-connectors-v2/connector-jdbc/pom.xml index 9721870da9d..bd22d6758ed 100644 --- a/seatunnel-connectors-v2/connector-jdbc/pom.xml +++ b/seatunnel-connectors-v2/connector-jdbc/pom.xml @@ -36,6 +36,7 @@ 9.2.1.jre8 5.2.5-HBase-2.x 12.2.0.1 + db2jcc4 @@ -109,8 +110,9 @@ ojdbc8 - com.ibm.db2 - jcc + com.ibm.db2.jcc + db2jcc + ${db2.version} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java index adb09419de2..cb33af160a7 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java @@ -23,18 +23,18 @@ public class DB2Dialect implements JdbcDialect { - @Override - public String dialectName() { - return "DB2"; - } + @Override + public String dialectName() { + return "DB2"; + } - @Override - public JdbcRowConverter getRowConverter() { - return new DB2JdbcRowConverter(); - } + @Override + public JdbcRowConverter getRowConverter() { + return new DB2JdbcRowConverter(); + } - @Override - public JdbcDialectTypeMapper getJdbcDialectTypeMapper() { - return new DB2TypeMapper(); - } + @Override + public JdbcDialectTypeMapper getJdbcDialectTypeMapper() { + return new DB2TypeMapper(); + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java index 6f629976c3e..deb269a5c89 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectFactory.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2; - import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectFactory; diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java index 80076c193d7..28cc0d02def 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java @@ -20,11 +20,13 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; + import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.AbstractJdbcRowConverter; public class DB2JdbcRowConverter extends AbstractJdbcRowConverter { + @Override public String converterName() { return "DB2"; diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java index 6745809e31d..a841254f81f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2; - import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; import org.apache.seatunnel.api.table.type.LocalTimeType; @@ -25,6 +24,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,12 +55,11 @@ public class DB2TypeMapper implements JdbcDialectTypeMapper { private static final String DB2_DOUBLE = "DOUBLE"; private static final String DB2_DOUBLE_PRECISION = "DOUBLE PRECISION"; private static final String DB2_DECFLOAT = "DECFLOAT"; - private static final String DB2_DECDOUBLE = "DECDOUBLE"; // string private static final String DB2_CHAR = "CHAR"; private static final String DB2_VARCHAR = "VARCHAR"; - private static final String DB2_CLOB = "CLOB"; private static final String DB2_LONG_VARCHAR = "LONG VARCHAR"; + private static final String DB2_CLOB = "CLOB"; // graphic private static final String DB2_GRAPHIC = "GRAPHIC"; private static final String DB2_VARGRAPHIC = "VARGRAPHIC"; @@ -79,13 +78,9 @@ public class DB2TypeMapper implements JdbcDialectTypeMapper { // ------------------------------blob------------------------- private static final String DB2_BLOB = "BLOB"; - private static final String DB2_NCHAR_MAPPING = "NCHAR_MAPPING"; // other private static final String DB2_XML = "XML"; - private static final String DB2_LOB = "LOB"; - private static final String DB2_DATALINK = "DATALINK"; - @SuppressWarnings("checkstyle:MagicNumber") @Override @@ -109,6 +104,7 @@ public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) th if (precision > 0) { return new DecimalType(precision, metadata.getScale(colIndex)); } + LOG.warn("decimal did define precision,scale, will be Decimal(38,18)"); return new DecimalType(38, 18); case DB2_REAL: return BasicType.FLOAT_TYPE; @@ -116,12 +112,11 @@ public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) th case DB2_DOUBLE: case DB2_DOUBLE_PRECISION: case DB2_DECFLOAT: - case DB2_DECDOUBLE: return BasicType.DOUBLE_TYPE; case DB2_CHAR: case DB2_VARCHAR: - case DB2_CLOB: case DB2_LONG_VARCHAR: + case DB2_CLOB: case DB2_GRAPHIC: case DB2_VARGRAPHIC: case DB2_LONG_VARGRAPHIC: @@ -130,7 +125,6 @@ public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) th case DB2_BINARY: case DB2_VARBINARY: case DB2_BLOB: - case DB2_NCHAR_MAPPING: return PrimitiveByteArrayType.INSTANCE; case DB2_DATE: return LocalTimeType.LOCAL_DATE_TYPE; @@ -139,10 +133,8 @@ public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) th case DB2_TIMESTAMP: return LocalTimeType.LOCAL_DATE_TIME_TYPE; case DB2_ROWID: - // should support + // maybe should support case DB2_XML: - case DB2_LOB: - case DB2_DATALINK: default: final String jdbcColumnName = metadata.getColumnName(colIndex); throw new UnsupportedOperationException( diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/jdbc/JdbcDb2IT.java new file mode 100644 index 00000000000..227fd9235d7 --- /dev/null +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/jdbc/JdbcDb2IT.java @@ -0,0 +1,142 @@ +package org.apache.seatunnel.e2e.flink.v2.jdbc; + +import static org.testcontainers.shaded.org.awaitility.Awaitility.given; + +import org.apache.seatunnel.e2e.flink.FlinkContainer; + +import com.google.common.collect.Lists; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +public class JdbcDb2IT extends FlinkContainer { + + private static final Logger LOG = LoggerFactory.getLogger(JdbcDb2IT.class); + /** + * db2 in dockerhub + */ + private static final String IMAGE = "ibmcom/db2:latest"; + private static final String HOST = "spark_e2e_db2"; + private static final int PORT = 50000; + private static final String LOCAL_HOST = "localhost"; + private static final int LOCAL_PORT = 50000; + private static final String USER = "DB2INST1"; + private static final String PASSWORD = "123456"; + private static final String DRIVER = "com.ibm.db2.jcc.DB2Driver"; + private static final String JDBC_URL = String.format("jdbc:db2://%s:%s/testdb", LOCAL_HOST, LOCAL_PORT); + + private static final String SOURCE_TABLE = "e2e_table_source"; + private static final String SINK_TABLE = "e2e_table_sink"; + + private GenericContainer server; + private Connection jdbcConnection; + + @BeforeEach + public void startDB2Container() throws ClassNotFoundException { + server = new GenericContainer<>(IMAGE) + .withNetwork(NETWORK) + .withNetworkAliases(HOST) + .withPrivilegedMode(true) + .withLogConsumer(new Slf4jLogConsumer(LOG)) + .withEnv("DB2INST1_PASSWORD", "123456") + .withEnv("DBNAME", "testdb") + .withEnv("LICENSE", "accept") + ; + server.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); + Startables.deepStart(Stream.of(server)).join(); + LOG.info("DB2 container started"); + Class.forName(DRIVER); + given().ignoreExceptions() + .await() + .atMost(180, TimeUnit.SECONDS) + .untilAsserted(this::initializeJdbcConnection); + initializeJdbcTable(); + LOG.info("db2 init success"); + } + + @AfterEach + public void closeGreenplumContainer() throws SQLException { + if (jdbcConnection != null) { + jdbcConnection.close(); + } + if (server != null) { + server.close(); + } + } + + private void initializeJdbcConnection() throws SQLException { + jdbcConnection = DriverManager.getConnection(JDBC_URL, USER, PASSWORD); + } + + /** + * init the table + */ + private void initializeJdbcTable() { + URL resource = JdbcDb2IT.class.getResource("/jdbc/init_sql/db2_init.conf"); + if (resource == null) { + throw new IllegalArgumentException("can't find find file"); + } + String file = resource.getFile(); + Config config = ConfigFactory.parseFile(new File(file)); + assert config.hasPath("table_source") && config.hasPath("DML") && config.hasPath("table_sink"); + try (Statement statement = jdbcConnection.createStatement()) { + // source + String sourceTableDDL = config.getString("table_source"); + statement.execute(sourceTableDDL); + LOG.info("source DDL success"); + String insertSQL = config.getString("DML"); + statement.execute(insertSQL); + LOG.info("source DML success"); + // sink + String sinkTableDDL = config.getString("table_sink"); + statement.execute(sinkTableDDL); + LOG.info("sink DDL success"); + } catch (SQLException e) { + throw new RuntimeException("Initializing table failed!", e); + } + } + + private void assertHasData(String table) { + try (Connection connection = DriverManager.getConnection(JDBC_URL, USER, PASSWORD)) { + Statement statement = connection.createStatement(); + String sql = String.format("select * from %s.%s limit 1", USER, table); + ResultSet source = statement.executeQuery(sql); + Assertions.assertTrue(source.next()); + } catch (SQLException e) { + throw new RuntimeException("server image error", e); + } + } + + @Test + void pullImageOK() { + assertHasData(SOURCE_TABLE); + } + + @Test + public void testJdbcSourceAndSink() throws IOException, InterruptedException { + assertHasData(SOURCE_TABLE); + Container.ExecResult execResult = executeSeaTunnelFlinkJob("/jdbc/jdbc_db2_source_and_sink.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + assertHasData(SINK_TABLE); + } +} diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/init_sql/db2_init.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/init_sql/db2_init.conf new file mode 100644 index 00000000000..ee6ac9a0c1e --- /dev/null +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/init_sql/db2_init.conf @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# db2 sql can't start with \n +table_source = """create table "DB2INST1".E2E_TABLE_SOURCE +( + COL_BOOLEAN BOOLEAN, + COL_INT INT, + COL_INTEGER INTEGER, + COL_SMALLINT SMALLINT, + COL_BIGINT BIGINT, + + COL_DECIMAL DECIMAL, + COL_DEC DEC, + COL_NUMERIC NUMERIC, + COL_NUMBER NUM, + + COL_TIMESTAMP TIMESTAMP, + COL_TIME TIME, + COL_DATE DATE, + + COL_REAL REAL, + COL_FLOAT FLOAT, + COL_DOUBLE_PRECISION DOUBLE PRECISION, + COL_DOUBLE DOUBLE, + COL_DECFLOAT DECFLOAT, + + COL_CHAR CHAR, + COL_VARCHAR VARCHAR(255), + COL_LONG_VARCHAR "LONG VARCHAR", + + + COL_GRAPHIC GRAPHIC(10), + COL_VARGRAPHIC VARGRAPHIC(1024), + COL_LONG_VARGRAPHIC "LONG VARGRAPHIC", + COL_DBCLOB DBCLOB, + + COL_BLOB BLOB, + COL_BINARY BINARY, + COL_VARBINARY VARBINARY(255), + COL_XML XML +) +""" + +table_sink = """create table "DB2INST1".E2E_TABLE_SINK +( + COL_BOOLEAN BOOLEAN, + COL_INT INT, + COL_INTEGER INTEGER, + COL_SMALLINT SMALLINT, + COL_BIGINT BIGINT, + + COL_DECIMAL DECIMAL, + COL_DEC DEC, + COL_NUMERIC NUMERIC, + COL_NUMBER NUM, + + COL_TIMESTAMP TIMESTAMP, + COL_TIME TIME, + COL_DATE DATE, + + COL_REAL REAL, + COL_FLOAT FLOAT, + COL_DOUBLE_PRECISION DOUBLE PRECISION, + COL_DOUBLE DOUBLE, + COL_DECFLOAT DECFLOAT, + + COL_CHAR CHAR, + COL_VARCHAR VARCHAR(255), + COL_LONG_VARCHAR "LONG VARCHAR", + + + COL_GRAPHIC GRAPHIC(10), + COL_VARGRAPHIC VARGRAPHIC(1024), + COL_LONG_VARGRAPHIC "LONG VARGRAPHIC", + COL_DBCLOB DBCLOB, + + COL_BLOB BLOB, + COL_BINARY BINARY, + COL_VARBINARY VARBINARY(255), + COL_XML XML +) +""" + +DML = """insert into "DB2INST1".E2E_TABLE_SOURCE + (COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, + COL_NUMERIC, COL_NUMBER, COL_TIMESTAMP, COL_TIME, COL_DATE, COL_REAL, COL_FLOAT, + COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, + COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC, + COL_DBCLOB, COL_BLOB, COL_BINARY, COL_VARBINARY, COL_XML) +VALUES (true, 1, 2, 3, 4, 5.0, 6.0, 7.0, 8.0, '2022-09-11 14:58:06.000000', null, '2022-09-11', 1.1, 1.1, 1.1, 1.1, 1.1, + 'a', 'varchar', 'longvarchar', 'GRAPHIC', 'var graphic', 'log graphic', null, null, null, null, + null) +""" diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf new file mode 100644 index 00000000000..4060ea50b46 --- /dev/null +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + # You can set flink configuration here + execution.parallelism = 1 + job.mode = "BATCH" + #execution.checkpoint.interval = 10000 + #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + Jdbc { + driver = com.ibm.db2.jcc.DB2Driver + url = "jdbc:db2://spark_e2e_db2:50000/testdb" + user = DB2INST1 + password = 123456 + query = """select COL_BOOLEAN, + COL_INT, + COL_INTEGER, + COL_SMALLINT, + COL_BIGINT, + COL_DECIMAL, + COL_DEC, + COL_NUMERIC, + COL_NUMBER, + COL_TIMESTAMP, + COL_DATE, + COL_REAL, + COL_FLOAT, + COL_DOUBLE_PRECISION, + COL_DOUBLE, + COL_DECFLOAT, + COL_CHAR, + COL_VARCHAR, + COL_LONG_VARCHAR, + COL_CLOB, + COL_GRAPHIC, + COL_VARGRAPHIC, + COL_LONG_VARGRAPHIC + from "DB2INST1".E2E_TABLE_SOURCE; + """ + } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc +} + +transform { + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql +} + +sink { + Jdbc { + driver = com.ibm.db2.jcc.DB2Driver + url = "jdbc:db2://spark_e2e_db2:50000/testdb" + user = DB2INST1 + password = 123456 + query = """insert into DB2INST1.E2E_TABLE_SOURCE(COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, + COL_NUMERIC, COL_NUMBER, COL_TIMESTAMP, COL_DATE, COL_REAL, COL_FLOAT, + COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, + COL_LONG_VARCHAR, COL_CLOB, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +""" + } + + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc +} diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml index ee7629f1498..cd86cbb81f7 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml @@ -96,6 +96,19 @@ com.microsoft.sqlserver mssql-jdbc + + com.ibm.db2.jcc + db2jcc + db2jcc4 + test + + + + org.testcontainers + db2 + 1.17.3 + test + diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java new file mode 100644 index 00000000000..d5a7c7acc3f --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java @@ -0,0 +1,150 @@ +package org.apache.seatunnel.e2e.spark.v2.jdbc; + +import static org.testcontainers.shaded.org.awaitility.Awaitility.given; + +import org.apache.seatunnel.e2e.spark.SparkContainer; + +import com.google.common.collect.Lists; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +public class JdbcDb2IT extends SparkContainer { + + private static final Logger LOG = LoggerFactory.getLogger(JdbcDb2IT.class); + /** + * db2 in dockerhub + */ + private static final String IMAGE = "ibmcom/db2:11.5.0.0"; + private static final String HOST = "spark_e2e_db2"; + private static final int PORT = 50000; + private static final String LOCAL_HOST = "localhost"; + private static final int LOCAL_PORT = 50001; + private static final String USER = "DB2INST1"; + private static final String PASSWORD = "123456"; + private static final String DRIVER = "com.ibm.db2.jcc.DB2Driver"; + private static final String JDBC_URL = String.format("jdbc:db2://%s:%s/testdb", LOCAL_HOST, LOCAL_PORT); + private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE"; + private static final String SINK_TABLE = "E2E_TABLE_SINK"; + + private GenericContainer server; + private Connection jdbcConnection; + + @BeforeEach + public void startDB2Container() throws ClassNotFoundException, SQLException { + server = new GenericContainer<>(IMAGE) + .withNetwork(NETWORK) + .withNetworkAliases(HOST) + .withPrivilegedMode(true) + .withLogConsumer(new Slf4jLogConsumer(LOG)) + .withEnv("DB2INST1_PASSWORD", "123456") + .withEnv("DBNAME", "testdb") + .withEnv("LICENSE", "accept") + ; + server.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); + Startables.deepStart(Stream.of(server)).join(); + LOG.info("DB2 container started"); + given().ignoreExceptions() + .await() + .atMost(180, TimeUnit.SECONDS) + .untilAsserted(this::initializeJdbcConnection); + initializeJdbcTable(); + } + + @AfterEach + public void closeGreenplumContainer() throws SQLException { + if (jdbcConnection != null) { + jdbcConnection.close(); + } + if (server != null) { + server.close(); + } + } + + private void initializeJdbcConnection() throws SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { + Properties properties = new Properties(); + properties.setProperty("user", USER); + properties.setProperty("password", PASSWORD); + properties.setProperty("com.ibm.db2.jcc.DB2BaseDataSource.retrieveMessagesFromServerOnGetMessage", "true"); + properties.setProperty("com.ibm.db2.jcc.DB2BaseDataSource.maxConnCachedParamBufferSize", "16"); + properties.setProperty("com.ibm.db2.jcc.DB2BaseDataSource.connectionTimeout", "180"); + Driver driver = (Driver) Class.forName(DRIVER).newInstance(); + jdbcConnection = driver.connect(JDBC_URL, properties); + Statement statement = jdbcConnection.createStatement(); + ResultSet resultSet = statement.executeQuery("select 1 from SYSSTAT.TABLES"); + Assertions.assertTrue(resultSet.next()); + resultSet.close(); + statement.close(); + } + + /** + * init the table + */ + private void initializeJdbcTable() { + URL resource = JdbcDb2IT.class.getResource("/jdbc/init_sql/db2_init.conf"); + if (resource == null) { + throw new IllegalArgumentException("can't find find file"); + } + String file = resource.getFile(); + Config config = ConfigFactory.parseFile(new File(file)); + assert config.hasPath("table_source") && config.hasPath("DML") && config.hasPath("table_sink"); + try (Statement statement = jdbcConnection.createStatement()) { + // source + LOG.info("source DDL start"); + String sourceTableDDL = config.getString("table_source"); + statement.execute(sourceTableDDL); + LOG.info("source DML start"); + String insertSQL = config.getString("DML"); + statement.execute(insertSQL); + LOG.info("sink DDL start"); + String sinkTableDDL = config.getString("table_sink"); + statement.execute(sinkTableDDL); + } catch (SQLException e) { + throw new RuntimeException("Initializing table failed!", e); + } + } + + private void assertHasData(String table) throws SQLException { + try (Statement statement = jdbcConnection.createStatement()) { + String sql = String.format("select * from \"%s\".%s", USER, table); + ResultSet source = statement.executeQuery(sql); + Assertions.assertTrue(source.next(), "result is null when sql is " + sql); + } catch (SQLException e) { + throw new RuntimeException("server image error", e); + } + } + + @Test + void pullImageOK() throws SQLException { + assertHasData(SOURCE_TABLE); + } + + @Test + public void testJdbcSourceAndSink() throws IOException, InterruptedException, SQLException { + assertHasData(SOURCE_TABLE); + Container.ExecResult execResult = executeSeaTunnelSparkJob("/jdbc/jdbc_db2_source_and_sink.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + assertHasData(SINK_TABLE); + } +} diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/init_sql/db2_init.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/init_sql/db2_init.conf new file mode 100644 index 00000000000..3274f930622 --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/init_sql/db2_init.conf @@ -0,0 +1,109 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# db2 sql can't start with \n +table_source = """create table "DB2INST1".E2E_TABLE_SOURCE +( + COL_BOOLEAN BOOLEAN, + COL_INT INT, + COL_INTEGER INTEGER, + COL_SMALLINT SMALLINT, + COL_BIGINT BIGINT, + + COL_DECIMAL DECIMAL, + COL_DEC DEC, + COL_NUMERIC NUMERIC, + COL_NUMBER NUM, + + COL_TIMESTAMP TIMESTAMP, + COL_TIME TIME, + COL_DATE DATE, + + COL_REAL REAL, + COL_FLOAT FLOAT, + COL_DOUBLE_PRECISION DOUBLE PRECISION, + COL_DOUBLE DOUBLE, + COL_DECFLOAT DECFLOAT, + + COL_CHAR CHAR, + COL_VARCHAR VARCHAR(255), + COL_LONG_VARCHAR "LONG VARCHAR", + + + COL_GRAPHIC GRAPHIC(10), + COL_VARGRAPHIC VARGRAPHIC(1024), + COL_LONG_VARGRAPHIC "LONG VARGRAPHIC", + COL_DBCLOB DBCLOB, + + COL_BLOB BLOB, + COL_BINARY BINARY, + COL_VARBINARY VARBINARY(255), + COL_XML XML +) +""" + +table_sink = """ +create table "DB2INST1".E2E_TABLE_SINK +( + COL_BOOLEAN BOOLEAN, + COL_INT INT, + COL_INTEGER INTEGER, + COL_SMALLINT SMALLINT, + COL_BIGINT BIGINT, + + COL_DECIMAL DECIMAL, + COL_DEC DEC, + COL_NUMERIC NUMERIC, + COL_NUMBER NUM, + + COL_TIMESTAMP TIMESTAMP, + COL_TIME TIME, + COL_DATE DATE, + + COL_REAL REAL, + COL_FLOAT FLOAT, + COL_DOUBLE_PRECISION DOUBLE PRECISION, + COL_DOUBLE DOUBLE, + COL_DECFLOAT DECFLOAT, + + COL_CHAR CHAR, + COL_VARCHAR VARCHAR(255), + COL_LONG_VARCHAR "LONG VARCHAR", + + + COL_GRAPHIC GRAPHIC(10), + COL_VARGRAPHIC VARGRAPHIC(1024), + COL_LONG_VARGRAPHIC "LONG VARGRAPHIC", + COL_DBCLOB DBCLOB, + + COL_BLOB BLOB, + COL_BINARY BINARY, + COL_VARBINARY VARBINARY(255), + COL_XML XML +) +""" + +DML = """insert into "DB2INST1".E2E_TABLE_SOURCE + (COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, + COL_NUMERIC, COL_NUMBER, COL_TIMESTAMP, COL_TIME, COL_DATE, COL_REAL, COL_FLOAT, + COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, + COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC, + COL_DBCLOB, COL_BLOB, COL_BINARY, COL_VARBINARY, COL_XML) +VALUES (true, 1, 2, 3, 4, 5.0, 6.0, 7.0, 8.0, '2022-09-11 14:58:06.000000', null, '2022-09-11', 1.1, 1.1, 1.1, 1.1, 1.1, + 'a', 'varchar', 'longvarchar', 'GRAPHIC', 'var graphic', 'log graphic', null, null, null, null, + null) +""" diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf new file mode 100644 index 00000000000..5ca4a623415 --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local + job.mode = "BATCH" +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + Jdbc { + driver = com.ibm.db2.jcc.DB2Driver + url = "jdbc:db2://spark_e2e_db2:50000/testdb" + user = DB2INST1 + password = 123456 + query = """ + select COL_BOOLEAN, + COL_INT, + COL_INTEGER, + COL_SMALLINT, + COL_BIGINT, + COL_DECIMAL, + COL_DEC, + COL_NUMERIC, + COL_NUMBER, + COL_REAL, + COL_FLOAT, + COL_DOUBLE_PRECISION, + COL_DOUBLE, + COL_DECFLOAT, + COL_CHAR, + COL_VARCHAR, + COL_LONG_VARCHAR, + COL_GRAPHIC, + COL_VARGRAPHIC, + COL_LONG_VARGRAPHIC + from "DB2INST1".E2E_TABLE_SOURCE; + """ + } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc +} + +transform { + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql +} + +sink { + Jdbc { + driver = com.ibm.db2.jcc.DB2Driver + url = "jdbc:db2://spark_e2e_db2:50000/testdb" + user = DB2INST1 + password = 123456 + query = """ + insert into "DB2INST1".E2E_TABLE_SINK(COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, + COL_NUMERIC, COL_NUMBER, COL_REAL, COL_FLOAT, + COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, + COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +""" + } + + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc +} diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml b/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml index 23f22f062ac..bef93603017 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml @@ -99,6 +99,11 @@ lz4 1.3.0 + + org.apache.seatunnel + connector-jdbc + 2.1.3-SNAPSHOT + - \ No newline at end of file + diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/jdbc_db2_source_and_sink.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/jdbc_db2_source_and_sink.conf new file mode 100644 index 00000000000..ac42d635f41 --- /dev/null +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/jdbc_db2_source_and_sink.conf @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local + job.mode = "BATCH" +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + Jdbc { + driver = com.ibm.db2.jcc.DB2Driver + url = "jdbc:db2://localhost:50000/testdb" + user = DB2INST1 + password = 123456 + query = """ + select COL_BOOLEAN, + COL_INT, + COL_INTEGER, + COL_SMALLINT, + COL_BIGINT, + COL_DECIMAL, + COL_DEC, + COL_NUMERIC, + COL_NUMBER, + COL_REAL, + COL_FLOAT, + COL_DOUBLE_PRECISION, + COL_DOUBLE, + COL_DECFLOAT, + COL_CHAR, + COL_VARCHAR, + COL_LONG_VARCHAR, + COL_GRAPHIC, + COL_VARGRAPHIC, + COL_LONG_VARGRAPHIC + from "DB2INST1".E2E_TABLE_SOURCE; + """ + } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc +} + +transform { + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql +} + +sink { + Jdbc { + driver = com.ibm.db2.jcc.DB2Driver + url = "jdbc:db2://localhost:50000/testdb" + user = DB2INST1 + password = 123456 + query = """ + insert into "DB2INST1".E2E_TABLE_SINK(COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, + COL_NUMERIC, COL_NUMBER, COL_REAL, COL_FLOAT, + COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, + COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +""" + } + + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc +} From 6dd85f60e1f4f9e48ea319067035bb41207ad8ec Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Tue, 13 Sep 2022 00:18:21 +0800 Subject: [PATCH 05/23] db2 e2e --- .../dialect/db2/DB2JdbcRowConverter.java | 8 ++++---- .../seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java | 16 +++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java index 28cc0d02def..688ea3ff418 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2JdbcRowConverter.java @@ -17,14 +17,14 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; - import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.AbstractJdbcRowConverter; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + public class DB2JdbcRowConverter extends AbstractJdbcRowConverter { @Override diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java index d5a7c7acc3f..12e1fb922b8 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java @@ -36,18 +36,18 @@ public class JdbcDb2IT extends SparkContainer { /** * db2 in dockerhub */ - private static final String IMAGE = "ibmcom/db2:11.5.0.0"; + private static final String IMAGE = "ibmcom/db2:latest"; private static final String HOST = "spark_e2e_db2"; private static final int PORT = 50000; - private static final String LOCAL_HOST = "localhost"; private static final int LOCAL_PORT = 50001; private static final String USER = "DB2INST1"; private static final String PASSWORD = "123456"; private static final String DRIVER = "com.ibm.db2.jcc.DB2Driver"; - private static final String JDBC_URL = String.format("jdbc:db2://%s:%s/testdb", LOCAL_HOST, LOCAL_PORT); + + private static final String DATABASE = "testdb"; private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE"; private static final String SINK_TABLE = "E2E_TABLE_SINK"; - + private String JDBC_URL; private GenericContainer server; private Connection jdbcConnection; @@ -58,12 +58,13 @@ public void startDB2Container() throws ClassNotFoundException, SQLException { .withNetworkAliases(HOST) .withPrivilegedMode(true) .withLogConsumer(new Slf4jLogConsumer(LOG)) - .withEnv("DB2INST1_PASSWORD", "123456") - .withEnv("DBNAME", "testdb") + .withEnv("DB2INST1_PASSWORD", PASSWORD) + .withEnv("DBNAME", DATABASE) .withEnv("LICENSE", "accept") ; server.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); Startables.deepStart(Stream.of(server)).join(); + JDBC_URL = String.format("jdbc:db2://%s:%s/%s", server.getContainerIpAddress(), LOCAL_PORT, DATABASE); LOG.info("DB2 container started"); given().ignoreExceptions() .await() @@ -86,9 +87,6 @@ private void initializeJdbcConnection() throws SQLException, ClassNotFoundExcept Properties properties = new Properties(); properties.setProperty("user", USER); properties.setProperty("password", PASSWORD); - properties.setProperty("com.ibm.db2.jcc.DB2BaseDataSource.retrieveMessagesFromServerOnGetMessage", "true"); - properties.setProperty("com.ibm.db2.jcc.DB2BaseDataSource.maxConnCachedParamBufferSize", "16"); - properties.setProperty("com.ibm.db2.jcc.DB2BaseDataSource.connectionTimeout", "180"); Driver driver = (Driver) Class.forName(DRIVER).newInstance(); jdbcConnection = driver.connect(JDBC_URL, properties); Statement statement = jdbcConnection.createStatement(); From 73747919094cc7b3fb2df41c24608fc56558296b Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 16 Oct 2022 11:07:51 +0800 Subject: [PATCH 06/23] fix db2IT --- seatunnel-config/README.md | 25 - seatunnel-config/pom.xml | 37 -- .../seatunnel-config-base/pom.xml | 135 ----- .../seatunnel-config-shade/pom.xml | 82 --- .../typesafe/config/ConfigParseOptions.java | 250 -------- .../typesafe/config/impl/ConfigNodePath.java | 57 -- .../typesafe/config/impl/ConfigParser.java | 569 ------------------ .../shade/com/typesafe/config/impl/Path.java | 242 -------- .../com/typesafe/config/impl/PathParser.java | 296 --------- .../config/impl/SimpleConfigObject.java | 568 ----------------- .../apache/seatunnel/config/CompleteTest.java | 54 -- .../seatunnel/config/ConfigFactoryTest.java | 83 --- .../seatunnel/config/JsonFormatTest.java | 53 -- .../seatunnel/config/utils/FileUtils.java | 39 -- .../src/test/resources/factory/config.conf | 66 -- .../src/test/resources/json/spark.batch.conf | 72 --- .../src/test/resources/json/spark.batch.json | 20 - .../test/resources/seatunnel/variables.conf | 62 -- .../connectors/seatunnel}/jdbc/JdbcDb2IT.java | 50 +- .../src/test/resources/init}/db2_init.conf | 0 .../e2e/flink/v2/jdbc/JdbcDb2IT.java | 142 ----- 21 files changed, 28 insertions(+), 2874 deletions(-) delete mode 100644 seatunnel-config/README.md delete mode 100644 seatunnel-config/pom.xml delete mode 100644 seatunnel-config/seatunnel-config-base/pom.xml delete mode 100644 seatunnel-config/seatunnel-config-shade/pom.xml delete mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/ConfigParseOptions.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigNodePath.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigParser.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/Path.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PathParser.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/SimpleConfigObject.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/CompleteTest.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/JsonFormatTest.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/utils/FileUtils.java delete mode 100644 seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf delete mode 100644 seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.conf delete mode 100644 seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.json delete mode 100644 seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf rename seatunnel-e2e/{seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2 => seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel}/jdbc/JdbcDb2IT.java (76%) rename seatunnel-e2e/{seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/init_sql => seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/init}/db2_init.conf (100%) delete mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/jdbc/JdbcDb2IT.java diff --git a/seatunnel-config/README.md b/seatunnel-config/README.md deleted file mode 100644 index 4932a3f2c9b..00000000000 --- a/seatunnel-config/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Introduction -The `seatunnel-config` is used to parse `seatunnel.conf` files. This module is based on `com.typesafe.config`, -We have made some enhancement and import our enhancement by using maven shade. Most of the times, you don't need to directly -using this module, since you can receive from maven repository. - -# How to modify the config module -If you want to modify the config module, you can follow the steps below. -1. Open the `seatunnel-config` module. -```xml - -``` -Open the annuotaion in `pom.xml` file, to import the `seatunnel-config` module. -2. Replace the `config-shade` dependency to project. -```xml - - org.apache.seatunnel - seatunnel-config-shade - ${project.version} - -``` -Add `${project.version}` to `seatunnel-config-shade` everywhere you use. \ No newline at end of file diff --git a/seatunnel-config/pom.xml b/seatunnel-config/pom.xml deleted file mode 100644 index 29a8a5e0cb2..00000000000 --- a/seatunnel-config/pom.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - org.apache.seatunnel - seatunnel - ${revision} - - 4.0.0 - seatunnel-config - pom - - seatunnel-config - - - seatunnel-config-shade - seatunnel-config-base - - diff --git a/seatunnel-config/seatunnel-config-base/pom.xml b/seatunnel-config/seatunnel-config-base/pom.xml deleted file mode 100644 index 5401d09c304..00000000000 --- a/seatunnel-config/seatunnel-config-base/pom.xml +++ /dev/null @@ -1,135 +0,0 @@ - - - - 4.0.0 - - org.apache.seatunnel - seatunnel-config - ${revision} - ../pom.xml - - seatunnel-config-base - seatunnel-config-base - - UTF-8 - ${java.version} - ${java.version} - true - org.apache.seatunnel.shade - - - - - com.typesafe - config - - - - - ${project.artifactId}-${project.version} - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - - org.apache.maven.plugins - maven-shade-plugin - - true - true - true - false - false - - - com.typesafe:config - - ** - - - META-INF/MANIFEST.MF - META-INF/NOTICE - com/typesafe/config/ConfigParseOptions.class - com/typesafe/config/impl/ConfigParser.class - com/typesafe/config/impl/ConfigNodePath.class - com/typesafe/config/impl/PathParser.class - com/typesafe/config/impl/Path.class - com/typesafe/config/impl/SimpleConfigObject.class - - - - - - com.typesafe.config - ${seatunnel.shade.package}.com.typesafe.config - - - - - - - - - - package - - shade - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - compile - package - - attach-artifact - - - - - ${basedir}/target/${project.artifactId}-${project.version}.jar - jar - optional - - - - - - - - - - \ No newline at end of file diff --git a/seatunnel-config/seatunnel-config-shade/pom.xml b/seatunnel-config/seatunnel-config-shade/pom.xml deleted file mode 100644 index 417d6e2de85..00000000000 --- a/seatunnel-config/seatunnel-config-shade/pom.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - - org.apache.seatunnel - seatunnel-config - ${revision} - ../pom.xml - - 4.0.0 - seatunnel-config-shade - - seatunnel-config-shade - - - UTF-8 - ${java.version} - ${java.version} - true - org.apache.seatunnel.shade - - - - org.apache.seatunnel - seatunnel-config-base - ${project.version} - - - org.scala-lang - scala-library - - - - - - ${project.artifactId}-${project.version} - - - - org.codehaus.mojo - build-helper-maven-plugin - - - compile - package - - attach-artifact - - - - - ${basedir}/target/${project.artifactId}-${project.version}.jar - jar - optional - - - - - - - - - - diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/ConfigParseOptions.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/ConfigParseOptions.java deleted file mode 100644 index b7949670257..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/ConfigParseOptions.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright (C) 2011-2012 Typesafe Inc. - */ - -package org.apache.seatunnel.shade.com.typesafe.config; - -/** - * A set of options related to parsing. - * - *

- * This object is immutable, so the "setters" return a new object. - * - *

- * Here is an example of creating a custom {@code ConfigParseOptions}: - * - * 

- *     ConfigParseOptions options = ConfigParseOptions.defaults()
- *         .setSyntax(ConfigSyntax.JSON)
- *         .setAllowMissing(false)
- *
- */ -public final class ConfigParseOptions { - - /** - * a.b.c - * a->b->c - */ - public static final String PATH_TOKEN_SEPARATOR = "->"; - - final ConfigSyntax syntax; - final String originDescription; - final boolean allowMissing; - final ConfigIncluder includer; - final ClassLoader classLoader; - - private ConfigParseOptions(ConfigSyntax syntax, String originDescription, boolean allowMissing, - ConfigIncluder includer, ClassLoader classLoader) { - this.syntax = syntax; - this.originDescription = originDescription; - this.allowMissing = allowMissing; - this.includer = includer; - this.classLoader = classLoader; - } - - /** - * Gets an instance of {@code ConfigParseOptions} with all fields - * set to the default values. Start with this instance and make any - * changes you need. - * - * @return the default parse options - */ - public static ConfigParseOptions defaults() { - return new ConfigParseOptions(null, null, true, null, null); - } - - /** - * Set the file format. If set to null, try to guess from any available - * filename extension; if guessing fails, assume {@link ConfigSyntax#CONF}. - * - * @param syntax a syntax or {@code null} for best guess - * @return options with the syntax set - */ - public ConfigParseOptions setSyntax(ConfigSyntax syntax) { - if (this.syntax == syntax) { - return this; - } else { - return new ConfigParseOptions(syntax, this.originDescription, this.allowMissing, - this.includer, this.classLoader); - } - } - - /** - * Gets the current syntax option, which may be null for "any". - * - * @return the current syntax or null - */ - public ConfigSyntax getSyntax() { - return syntax; - } - - /** - * Set a description for the thing being parsed. In most cases this will be - * set up for you to something like the filename, but if you provide just an - * input stream you might want to improve on it. Set to null to allow the - * library to come up with something automatically. This description is the - * basis for the {@link ConfigOrigin} of the parsed values. - * - * @param originDescription description to put in the {@link ConfigOrigin} - * @return options with the origin description set - */ - public ConfigParseOptions setOriginDescription(String originDescription) { - // findbugs complains about == here but is wrong, do not "fix" - if (this.originDescription == originDescription) { - return this; - } else if (this.originDescription != null && originDescription != null - && this.originDescription.equals(originDescription)) { - return this; - } else { - return new ConfigParseOptions(this.syntax, originDescription, this.allowMissing, - this.includer, this.classLoader); - } - } - - /** - * Gets the current origin description, which may be null for "automatic". - * - * @return the current origin description or null - */ - public String getOriginDescription() { - return originDescription; - } - - /** - * this is package-private, not public API - */ - ConfigParseOptions withFallbackOriginDescription(String originDescription) { - if (this.originDescription == null) { - return setOriginDescription(originDescription); - } else { - return this; - } - } - - /** - * Set to false to throw an exception if the item being parsed (for example - * a file) is missing. Set to true to just return an empty document in that - * case. Note that this setting applies on only to fetching the root document, - * it has no effect on any nested includes. - * - * @param allowMissing true to silently ignore missing item - * @return options with the "allow missing" flag set - */ - public ConfigParseOptions setAllowMissing(boolean allowMissing) { - if (this.allowMissing == allowMissing) { - return this; - } else { - return new ConfigParseOptions(this.syntax, this.originDescription, allowMissing, - this.includer, this.classLoader); - } - } - - /** - * Gets the current "allow missing" flag. - * - * @return whether we allow missing files - */ - public boolean getAllowMissing() { - return allowMissing; - } - - /** - * Set a {@link ConfigIncluder} which customizes how includes are handled. - * null means to use the default includer. - * - * @param includer the includer to use or null for default - * @return new version of the parse options with different includer - */ - public ConfigParseOptions setIncluder(ConfigIncluder includer) { - if (this.includer == includer) { - return this; - } else { - return new ConfigParseOptions(this.syntax, this.originDescription, this.allowMissing, - includer, this.classLoader); - } - } - - /** - * Prepends a {@link ConfigIncluder} which customizes how - * includes are handled. To prepend your includer, the - * library calls {@link ConfigIncluder#withFallback} on your - * includer to append the existing includer to it. - * - * @param includer the includer to prepend (may not be null) - * @return new version of the parse options with different includer - */ - public ConfigParseOptions prependIncluder(ConfigIncluder includer) { - if (includer == null) { - throw new NullPointerException("null includer passed to prependIncluder"); - } - if (this.includer == includer) { - return this; - } else if (this.includer != null) { - return setIncluder(includer.withFallback(this.includer)); - } else { - return setIncluder(includer); - } - } - - /** - * Appends a {@link ConfigIncluder} which customizes how - * includes are handled. To append, the library calls {@link - * ConfigIncluder#withFallback} on the existing includer. - * - * @param includer the includer to append (may not be null) - * @return new version of the parse options with different includer - */ - public ConfigParseOptions appendIncluder(ConfigIncluder includer) { - if (includer == null) { - throw new NullPointerException("null includer passed to appendIncluder"); - } - if (this.includer == includer) { - return this; - } else if (this.includer != null) { - return setIncluder(this.includer.withFallback(includer)); - } else { - return setIncluder(includer); - } - } - - /** - * Gets the current includer (will be null for the default includer). - * - * @return current includer or null - */ - public ConfigIncluder getIncluder() { - return includer; - } - - /** - * Set the class loader. If set to null, - * {@code Thread.currentThread().getContextClassLoader()} will be used. - * - * @param loader a class loader or {@code null} to use thread context class - * loader - * @return options with the class loader set - */ - public ConfigParseOptions setClassLoader(ClassLoader loader) { - if (this.classLoader == loader) { - return this; - } else { - return new ConfigParseOptions(this.syntax, this.originDescription, this.allowMissing, - this.includer, loader); - } - } - - /** - * Get the class loader; never returns {@code null}, if the class loader was - * unset, returns - * {@code Thread.currentThread().getContextClassLoader()}. - * - * @return class loader to use - */ - public ClassLoader getClassLoader() { - if (this.classLoader == null) { - return Thread.currentThread().getContextClassLoader(); - } else { - return this.classLoader; - } - } -} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigNodePath.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigNodePath.java deleted file mode 100644 index d917d84e849..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigNodePath.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2011-2012 Typesafe Inc. - */ - -package org.apache.seatunnel.shade.com.typesafe.config.impl; - -import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions; - -import java.util.ArrayList; -import java.util.Collection; - -final class ConfigNodePath extends AbstractConfigNode { - private final Path path; - final ArrayList tokens; - - ConfigNodePath(Path path, Collection tokens) { - this.path = path; - this.tokens = new ArrayList<>(tokens); - } - - @Override - protected Collection tokens() { - return tokens; - } - - protected Path value() { - return path; - } - - protected ConfigNodePath subPath(int toRemove) { - int periodCount = 0; - ArrayList tokensCopy = new ArrayList<>(tokens); - for (int i = 0; i < tokensCopy.size(); i++) { - if (Tokens.isUnquotedText(tokensCopy.get(i)) && - tokensCopy.get(i).tokenText().equals(ConfigParseOptions.PATH_TOKEN_SEPARATOR)) { - periodCount++; - } - - if (periodCount == toRemove) { - return new ConfigNodePath(path.subPath(toRemove), tokensCopy.subList(i + 1, tokensCopy.size())); - } - } - throw new ConfigException.BugOrBroken("Tried to remove too many elements from a Path node"); - } - - protected ConfigNodePath first() { - ArrayList tokensCopy = new ArrayList<>(tokens); - for (int i = 0; i < tokensCopy.size(); i++) { - if (Tokens.isUnquotedText(tokensCopy.get(i)) && - tokensCopy.get(i).tokenText().equals(ConfigParseOptions.PATH_TOKEN_SEPARATOR)) { - return new ConfigNodePath(path.subPath(0, 1), tokensCopy.subList(0, i)); - } - } - return this; - } -} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigParser.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigParser.java deleted file mode 100644 index 39b9cb5d463..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigParser.java +++ /dev/null @@ -1,569 +0,0 @@ -/* - * Copyright (C) 2011-2012 Typesafe Inc. - */ - -package org.apache.seatunnel.shade.com.typesafe.config.impl; - -import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigIncludeContext; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigOrigin; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigSyntax; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; - -import java.io.File; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; - -final class ConfigParser { - static AbstractConfigValue parse(ConfigNodeRoot document, - ConfigOrigin origin, ConfigParseOptions options, - ConfigIncludeContext includeContext) { - ParseContext context = new ParseContext(options.getSyntax(), origin, document, - SimpleIncluder.makeFull(options.getIncluder()), includeContext); - return context.parse(); - } - - private static final class ParseContext { - private int lineNumber; - private final ConfigNodeRoot document; - private final FullIncluder includer; - private final ConfigIncludeContext includeContext; - private final ConfigSyntax flavor; - private final ConfigOrigin baseOrigin; - private final LinkedList pathStack; - - // the number of lists we are inside; this is used to detect the "cannot - // generate a reference to a list element" problem, and once we fix that - // problem we should be able to get rid of this variable. - int arrayCount; - - ParseContext(ConfigSyntax flavor, ConfigOrigin origin, ConfigNodeRoot document, - FullIncluder includer, ConfigIncludeContext includeContext) { - lineNumber = 1; - this.document = document; - this.flavor = flavor; - this.baseOrigin = origin; - this.includer = includer; - this.includeContext = includeContext; - this.pathStack = new LinkedList<>(); - this.arrayCount = 0; - } - - // merge a bunch of adjacent values into one - // value; change unquoted text into a string - // value. - private AbstractConfigValue parseConcatenation(ConfigNodeConcatenation n) { - // this trick is not done in JSON - if (flavor == ConfigSyntax.JSON) { - throw new ConfigException.BugOrBroken("Found a concatenation node in JSON"); - } - - List values = new ArrayList<>(); - - for (AbstractConfigNode node : n.children()) { - AbstractConfigValue v = null; - if (node instanceof AbstractConfigNodeValue) { - v = parseValue((AbstractConfigNodeValue) node, null); - values.add(v); - } - } - - return ConfigConcatenation.concatenate(values); - } - - private SimpleConfigOrigin lineOrigin() { - return ((SimpleConfigOrigin) baseOrigin).withLineNumber(lineNumber); - } - - private ConfigException parseError(String message) { - return parseError(message, null); - } - - private ConfigException parseError(String message, Throwable cause) { - return new ConfigException.Parse(lineOrigin(), message, cause); - } - - private Path fullCurrentPath() { - // pathStack has top of stack at front - if (pathStack.isEmpty()) { - throw new ConfigException.BugOrBroken("Bug in parser; tried to get current path when at root"); - } else { - return new Path(pathStack.descendingIterator()); - } - } - - private AbstractConfigValue parseValue(AbstractConfigNodeValue n, List comments) { - AbstractConfigValue v; - - int startingArrayCount = arrayCount; - - if (n instanceof ConfigNodeSimpleValue) { - v = ((ConfigNodeSimpleValue) n).value(); - } else if (n instanceof ConfigNodeObject) { - - Path path = pathStack.peekFirst(); - - if (path != null && !ConfigSyntax.JSON.equals(flavor) - && ("source".equals(path.first()) - || "transform".equals(path.first()) - || "sink".equals(path.first()))) { - v = parseObjectForSeatunnel((ConfigNodeObject) n); - } else { - v = parseObject((ConfigNodeObject) n); - } - - } else if (n instanceof ConfigNodeArray) { - v = parseArray((ConfigNodeArray) n); - } else if (n instanceof ConfigNodeConcatenation) { - v = parseConcatenation((ConfigNodeConcatenation) n); - } else { - throw parseError("Expecting a value but got wrong node type: " + n.getClass()); - } - - if (comments != null && !comments.isEmpty()) { - v = v.withOrigin(v.origin().prependComments(new ArrayList<>(comments))); - comments.clear(); - } - - if (arrayCount != startingArrayCount) { - throw new ConfigException.BugOrBroken("Bug in config parser: unbalanced array count"); - } - - return v; - } - - private static AbstractConfigObject createValueUnderPath(Path path, - AbstractConfigValue value) { - // for path foo.bar, we are creating - // { "foo" : { "bar" : value } } - List keys = new ArrayList<>(); - - String key = path.first(); - Path remaining = path.remainder(); - while (key != null) { - keys.add(key); - if (remaining == null) { - break; - } else { - key = remaining.first(); - remaining = remaining.remainder(); - } - } - - // the withComments(null) is to ensure comments are only - // on the exact leaf node they apply to. - // a comment before "foo.bar" applies to the full setting - // "foo.bar" not also to "foo" - ListIterator i = keys.listIterator(keys.size()); - String deepest = i.previous(); - AbstractConfigObject o = new SimpleConfigObject(value.origin().withComments(null), - Collections.singletonMap(deepest, value)); - while (i.hasPrevious()) { - Map m = Collections.singletonMap(i.previous(), o); - o = new SimpleConfigObject(value.origin().withComments(null), m); - } - - return o; - } - - private void parseInclude(Map values, ConfigNodeInclude n) { - boolean isRequired = n.isRequired(); - ConfigIncludeContext cic = includeContext.setParseOptions(includeContext.parseOptions().setAllowMissing(!isRequired)); - - AbstractConfigObject obj; - switch (n.kind()) { - case URL: - URL url; - try { - url = new URL(n.name()); - } catch (MalformedURLException e) { - throw parseError("include url() specifies an invalid URL: " + n.name(), e); - } - obj = (AbstractConfigObject) includer.includeURL(cic, url); - break; - - case FILE: - obj = (AbstractConfigObject) includer.includeFile(cic, - new File(n.name())); - break; - - case CLASSPATH: - obj = (AbstractConfigObject) includer.includeResources(cic, n.name()); - break; - - case HEURISTIC: - obj = (AbstractConfigObject) includer - .include(cic, n.name()); - break; - - default: - throw new ConfigException.BugOrBroken("should not be reached"); - } - - // we really should make this work, but for now throwing an - // exception is better than producing an incorrect result. - // See https://github.com/lightbend/config/issues/160 - if (arrayCount > 0 && obj.resolveStatus() != ResolveStatus.RESOLVED) { - throw parseError("Due to current limitations of the config parser, when an include statement is nested inside a list value, " - + "${} substitutions inside the included file cannot be resolved correctly. Either move the include outside of the list value or " - + "remove the ${} statements from the included file."); - } - - if (!pathStack.isEmpty()) { - Path prefix = fullCurrentPath(); - obj = obj.relativized(prefix); - } - - for (String key : obj.keySet()) { - AbstractConfigValue v = obj.get(key); - AbstractConfigValue existing = values.get(key); - if (existing != null) { - values.put(key, v.withFallback(existing)); - } else { - values.put(key, v); - } - } - } - - private SimpleConfigList parseObjectForSeatunnel(ConfigNodeObject n) { - - Map values = new LinkedHashMap<>(); - List valuesList = new ArrayList<>(); - SimpleConfigOrigin objectOrigin = lineOrigin(); - boolean lastWasNewline = false; - - ArrayList nodes = new ArrayList<>(n.children()); - List comments = new ArrayList<>(); - for (int i = 0; i < nodes.size(); i++) { - AbstractConfigNode node = nodes.get(i); - if (node instanceof ConfigNodeComment) { - lastWasNewline = false; - comments.add(((ConfigNodeComment) node).commentText()); - } else if (node instanceof ConfigNodeSingleToken && Tokens.isNewline(((ConfigNodeSingleToken) node).token())) { - lineNumber++; - if (lastWasNewline) { - // Drop all comments if there was a blank line and start a new comment block - comments.clear(); - } - lastWasNewline = true; - } else if (flavor != ConfigSyntax.JSON && node instanceof ConfigNodeInclude) { - parseInclude(values, (ConfigNodeInclude) node); - lastWasNewline = false; - } else if (node instanceof ConfigNodeField) { - lastWasNewline = false; - Path path = ((ConfigNodeField) node).path().value(); - comments.addAll(((ConfigNodeField) node).comments()); - - // path must be on-stack while we parse the value - pathStack.push(path); - if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) { - // we really should make this work, but for now throwing - // an exception is better than producing an incorrect - // result. See - // https://github.com/lightbend/config/issues/160 - if (arrayCount > 0) { - throw parseError("Due to current limitations of the config parser, += does not work nested inside a list. " - + "+= expands to a ${} substitution and the path in ${} cannot currently refer to list elements. " - + "You might be able to move the += outside of the list and then refer to it from inside the list with ${}."); - } - - // because we will put it in an array after the fact so - // we want this to be incremented during the parseValue - // below in order to throw the above exception. - arrayCount += 1; - } - - AbstractConfigNodeValue valueNode; - AbstractConfigValue newValue; - - valueNode = ((ConfigNodeField) node).value(); - - // comments from the key token go to the value token - newValue = parseValue(valueNode, comments); - - if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) { - arrayCount -= 1; - - List concat = new ArrayList<>(2); - AbstractConfigValue previousRef = new ConfigReference(newValue.origin(), - new SubstitutionExpression(fullCurrentPath(), true /* optional */)); - AbstractConfigValue list = new SimpleConfigList(newValue.origin(), - Collections.singletonList(newValue)); - concat.add(previousRef); - concat.add(list); - newValue = ConfigConcatenation.concatenate(concat); - } - - // Grab any trailing comments on the same line - if (i < nodes.size() - 1) { - i++; - while (i < nodes.size()) { - if (nodes.get(i) instanceof ConfigNodeComment) { - ConfigNodeComment comment = (ConfigNodeComment) nodes.get(i); - newValue = newValue.withOrigin(newValue.origin().appendComments( - Collections.singletonList(comment.commentText()))); - break; - } else if (nodes.get(i) instanceof ConfigNodeSingleToken) { - ConfigNodeSingleToken curr = (ConfigNodeSingleToken) nodes.get(i); - if (curr.token() == Tokens.COMMA || Tokens.isIgnoredWhitespace(curr.token())) { - // keep searching, as there could still be a comment - } else { - i--; - break; - } - } else { - i--; - break; - } - i++; - } - } - - pathStack.pop(); - - String key = path.first(); - Path remaining = path.remainder(); - - if (remaining == null) { - - Map m = Collections.singletonMap("plugin_name", key); - newValue = newValue.withFallback(ConfigValueFactory.fromMap(m)); - - values.put(key, newValue); - valuesList.add(newValue); - } else { - if (flavor == ConfigSyntax.JSON) { - throw new ConfigException.BugOrBroken( - "somehow got multi-element path in JSON mode"); - } - - AbstractConfigObject obj = createValueUnderPath( - remaining, newValue); - - Map m = Collections.singletonMap("plugin_name", key); - obj = obj.withFallback(ConfigValueFactory.fromMap(m)); - - values.put(key, obj); - valuesList.add(obj); - } - } - } - - return new SimpleConfigList(objectOrigin, valuesList); - } - - private AbstractConfigObject parseObject(ConfigNodeObject n) { - Map values = new LinkedHashMap<>(); - SimpleConfigOrigin objectOrigin = lineOrigin(); - boolean lastWasNewline = false; - - ArrayList nodes = new ArrayList<>(n.children()); - List comments = new ArrayList<>(); - for (int i = 0; i < nodes.size(); i++) { - AbstractConfigNode node = nodes.get(i); - if (node instanceof ConfigNodeComment) { - lastWasNewline = false; - comments.add(((ConfigNodeComment) node).commentText()); - } else if (node instanceof ConfigNodeSingleToken && Tokens.isNewline(((ConfigNodeSingleToken) node).token())) { - lineNumber++; - if (lastWasNewline) { - // Drop all comments if there was a blank line and start a new comment block - comments.clear(); - } - lastWasNewline = true; - } else if (flavor != ConfigSyntax.JSON && node instanceof ConfigNodeInclude) { - parseInclude(values, (ConfigNodeInclude) node); - lastWasNewline = false; - } else if (node instanceof ConfigNodeField) { - lastWasNewline = false; - Path path = ((ConfigNodeField) node).path().value(); - comments.addAll(((ConfigNodeField) node).comments()); - - // path must be on-stack while we parse the value - pathStack.push(path); - if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) { - // we really should make this work, but for now throwing - // an exception is better than producing an incorrect - // result. See - // https://github.com/lightbend/config/issues/160 - if (arrayCount > 0) { - throw parseError("Due to current limitations of the config parser, += does not work nested inside a list. " - + "+= expands to a ${} substitution and the path in ${} cannot currently refer to list elements. " - + "You might be able to move the += outside of the list and then refer to it from inside the list with ${}."); - } - - // because we will put it in an array after the fact so - // we want this to be incremented during the parseValue - // below in order to throw the above exception. - arrayCount += 1; - } - - AbstractConfigNodeValue valueNode; - AbstractConfigValue newValue; - - valueNode = ((ConfigNodeField) node).value(); - - // comments from the key token go to the value token - newValue = parseValue(valueNode, comments); - - if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) { - arrayCount -= 1; - - List concat = new ArrayList<>(2); - AbstractConfigValue previousRef = new ConfigReference(newValue.origin(), - new SubstitutionExpression(fullCurrentPath(), true /* optional */)); - AbstractConfigValue list = new SimpleConfigList(newValue.origin(), - Collections.singletonList(newValue)); - concat.add(previousRef); - concat.add(list); - newValue = ConfigConcatenation.concatenate(concat); - } - - // Grab any trailing comments on the same line - if (i < nodes.size() - 1) { - i++; - while (i < nodes.size()) { - if (nodes.get(i) instanceof ConfigNodeComment) { - ConfigNodeComment comment = (ConfigNodeComment) nodes.get(i); - newValue = newValue.withOrigin(newValue.origin().appendComments( - Collections.singletonList(comment.commentText()))); - break; - } else if (nodes.get(i) instanceof ConfigNodeSingleToken) { - ConfigNodeSingleToken curr = (ConfigNodeSingleToken) nodes.get(i); - if (curr.token() == Tokens.COMMA || Tokens.isIgnoredWhitespace(curr.token())) { - // keep searching, as there could still be a comment - } else { - i--; - break; - } - } else { - i--; - break; - } - i++; - } - } - - pathStack.pop(); - - String key = path.first(); - Path remaining = path.remainder(); - - if (remaining == null) { - AbstractConfigValue existing = values.get(key); - if (existing != null) { - // In strict JSON, dups should be an error; while in - // our custom config language, they should be merged - // if the value is an object (or substitution that - // could become an object). - - if (flavor == ConfigSyntax.JSON) { - throw parseError("JSON does not allow duplicate fields: '" - + key - + "' was already seen at " - + existing.origin().description()); - } else { - newValue = newValue.withFallback(existing); - } - } - values.put(key, newValue); - } else { - if (flavor == ConfigSyntax.JSON) { - throw new ConfigException.BugOrBroken( - "somehow got multi-element path in JSON mode"); - } - - AbstractConfigObject obj = createValueUnderPath( - remaining, newValue); - AbstractConfigValue existing = values.get(key); - if (existing != null) { - obj = obj.withFallback(existing); - } - values.put(key, obj); - } - } - } - - return new SimpleConfigObject(objectOrigin, values); - } - - private SimpleConfigList parseArray(ConfigNodeArray n) { - arrayCount += 1; - - SimpleConfigOrigin arrayOrigin = lineOrigin(); - List values = new ArrayList<>(); - - boolean lastWasNewLine = false; - List comments = new ArrayList<>(); - - AbstractConfigValue v = null; - - for (AbstractConfigNode node : n.children()) { - if (node instanceof ConfigNodeComment) { - comments.add(((ConfigNodeComment) node).commentText()); - lastWasNewLine = false; - } else if (node instanceof ConfigNodeSingleToken && Tokens.isNewline(((ConfigNodeSingleToken) node).token())) { - lineNumber++; - if (lastWasNewLine && v == null) { - comments.clear(); - } else if (v != null) { - values.add(v.withOrigin(v.origin().appendComments(new ArrayList<>(comments)))); - comments.clear(); - v = null; - } - lastWasNewLine = true; - } else if (node instanceof AbstractConfigNodeValue) { - lastWasNewLine = false; - if (v != null) { - values.add(v.withOrigin(v.origin().appendComments(new ArrayList<>(comments)))); - comments.clear(); - } - v = parseValue((AbstractConfigNodeValue) node, comments); - } - } - // There shouldn't be any comments at this point, but add them just in case - if (v != null) { - values.add(v.withOrigin(v.origin().appendComments(new ArrayList<>(comments)))); - } - arrayCount -= 1; - return new SimpleConfigList(arrayOrigin, values); - } - - AbstractConfigValue parse() { - AbstractConfigValue result = null; - ArrayList comments = new ArrayList<>(); - boolean lastWasNewLine = false; - for (AbstractConfigNode node : document.children()) { - if (node instanceof ConfigNodeComment) { - comments.add(((ConfigNodeComment) node).commentText()); - lastWasNewLine = false; - } else if (node instanceof ConfigNodeSingleToken) { - Token t = ((ConfigNodeSingleToken) node).token(); - if (Tokens.isNewline(t)) { - lineNumber++; - if (lastWasNewLine && result == null) { - comments.clear(); - } else if (result != null) { - result = result.withOrigin(result.origin().appendComments(new ArrayList<>(comments))); - comments.clear(); - break; - } - lastWasNewLine = true; - } - } else if (node instanceof ConfigNodeComplexValue) { - result = parseValue((ConfigNodeComplexValue) node, comments); - lastWasNewLine = false; - } - } - return result; - } - } -} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/Path.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/Path.java deleted file mode 100644 index 8ec318f2aa4..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/Path.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright (C) 2011-2012 Typesafe Inc. - */ - -package org.apache.seatunnel.shade.com.typesafe.config.impl; - -import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions; - -import java.util.Iterator; -import java.util.List; - -final class Path { - - private final String first; - private final Path remainder; - private static final int DEFAULT_VALUE = 41; - - Path(String first, Path remainder) { - this.first = first; - this.remainder = remainder; - } - - Path(String... elements) { - if (elements.length == 0) { - throw new ConfigException.BugOrBroken("empty path"); - } - this.first = elements[0]; - if (elements.length > 1) { - PathBuilder pb = new PathBuilder(); - for (int i = 1; i < elements.length; ++i) { - pb.appendKey(elements[i]); - } - this.remainder = pb.result(); - } else { - this.remainder = null; - } - } - - // append all the paths in the list together into one path - Path(List pathsToConcat) { - this(pathsToConcat.iterator()); - } - - // append all the paths in the iterator together into one path - Path(Iterator i) { - if (!i.hasNext()) { - throw new ConfigException.BugOrBroken("empty path"); - } - - Path firstPath = i.next(); - this.first = firstPath.first; - - PathBuilder pb = new PathBuilder(); - if (firstPath.remainder != null) { - pb.appendPath(firstPath.remainder); - } - while (i.hasNext()) { - pb.appendPath(i.next()); - } - this.remainder = pb.result(); - } - - String first() { - return first; - } - - /** - * @return path minus the first element or null if no more elements - */ - Path remainder() { - return remainder; - } - - /** - * @return path minus the last element or null if we have just one element - */ - Path parent() { - if (remainder == null) { - return null; - } - - PathBuilder pb = new PathBuilder(); - Path p = this; - while (p.remainder != null) { - pb.appendKey(p.first); - p = p.remainder; - } - return pb.result(); - } - - /** - * @return last element in the path - */ - String last() { - Path p = this; - while (p.remainder != null) { - p = p.remainder; - } - return p.first; - } - - Path prepend(Path toPrepend) { - PathBuilder pb = new PathBuilder(); - pb.appendPath(toPrepend); - pb.appendPath(this); - return pb.result(); - } - - int length() { - int count = 1; - Path p = remainder; - while (p != null) { - count += 1; - p = p.remainder; - } - return count; - } - - Path subPath(int removeFromFront) { - int count = removeFromFront; - Path p = this; - while (p != null && count > 0) { - count -= 1; - p = p.remainder; - } - return p; - } - - Path subPath(int firstIndex, int lastIndex) { - if (lastIndex < firstIndex) { - throw new ConfigException.BugOrBroken("bad call to subPath"); - } - - Path from = subPath(firstIndex); - PathBuilder pb = new PathBuilder(); - int count = lastIndex - firstIndex; - while (count > 0) { - count -= 1; - pb.appendKey(from.first()); - from = from.remainder(); - if (from == null) { - throw new ConfigException.BugOrBroken("subPath lastIndex out of range " + lastIndex); - } - } - return pb.result(); - } - - boolean startsWith(Path other) { - Path myRemainder = this; - Path otherRemainder = other; - if (otherRemainder.length() <= myRemainder.length()) { - while (otherRemainder != null) { - if (!otherRemainder.first().equals(myRemainder.first())) { - return false; - } - myRemainder = myRemainder.remainder(); - otherRemainder = otherRemainder.remainder(); - } - return true; - } - return false; - } - - @Override - public boolean equals(Object other) { - if (other instanceof Path) { - Path that = (Path) other; - return this.first.equals(that.first) - && ConfigImplUtil.equalsHandlingNull(this.remainder, - that.remainder); - } else { - return false; - } - } - - @Override - public int hashCode() { - return DEFAULT_VALUE * (DEFAULT_VALUE + first.hashCode()) - + (remainder == null ? 0 : remainder.hashCode()); - } - - // this doesn't have a very precise meaning, just to reduce - // noise from quotes in the rendered path for average cases - static boolean hasFunkyChars(String s) { - int length = s.length(); - - if (length == 0) { - return false; - } - - for (int i = 0; i < length; ++i) { - char c = s.charAt(i); - - if (Character.isLetterOrDigit(c) || c == '-' || c == '_' || c == '.') { - continue; - } else { - return true; - } - } - return false; - } - - private void appendToStringBuilder(StringBuilder sb) { - if (hasFunkyChars(first) || first.isEmpty()) { - sb.append(ConfigImplUtil.renderJsonString(first)); - } else { - sb.append(first); - } - if (remainder != null) { - sb.append(ConfigParseOptions.PATH_TOKEN_SEPARATOR); - remainder.appendToStringBuilder(sb); - } - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("Path("); - appendToStringBuilder(sb); - sb.append(")"); - return sb.toString(); - } - - /** - * toString() is a debugging-oriented version while this is an - * error-message-oriented human-readable one. - */ - String render() { - StringBuilder sb = new StringBuilder(); - appendToStringBuilder(sb); - return sb.toString(); - } - - static Path newKey(String key) { - return new Path(key, null); - } - - static Path newPath(String path) { - return PathParser.parsePath(path); - } -} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PathParser.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PathParser.java deleted file mode 100644 index b512cd1cc44..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PathParser.java +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright (C) 2011-2012 Typesafe Inc. - */ - -package org.apache.seatunnel.shade.com.typesafe.config.impl; - -import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigOrigin; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigSyntax; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; - -import java.io.StringReader; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -final class PathParser { - - static ConfigOrigin API_ORIGIN = SimpleConfigOrigin.newSimple("path parameter"); - - static ConfigNodePath parsePathNode(String path) { - return parsePathNode(path, ConfigSyntax.CONF); - } - - static ConfigNodePath parsePathNode(String path, ConfigSyntax flavor) { - try (StringReader reader = new StringReader(path)) { - Iterator tokens = Tokenizer.tokenize(API_ORIGIN, reader, flavor); - tokens.next(); // drop START - return parsePathNodeExpression(tokens, API_ORIGIN, path, flavor); - } - } - - static Path parsePath(String path) { - Path speculated = speculativeFastParsePath(path); - if (speculated != null) { - return speculated; - } - try (StringReader reader = new StringReader(path)) { - Iterator tokens = Tokenizer.tokenize(API_ORIGIN, reader, ConfigSyntax.CONF); - tokens.next(); // drop START - return parsePathExpression(tokens, API_ORIGIN, path); - } - } - - protected static Path parsePathExpression(Iterator expression, - ConfigOrigin origin) { - return parsePathExpression(expression, origin, null, null, ConfigSyntax.CONF); - } - - protected static Path parsePathExpression(Iterator expression, - ConfigOrigin origin, String originalText) { - return parsePathExpression(expression, origin, originalText, null, ConfigSyntax.CONF); - } - - protected static ConfigNodePath parsePathNodeExpression(Iterator expression, - ConfigOrigin origin) { - return parsePathNodeExpression(expression, origin, null, ConfigSyntax.CONF); - } - - protected static ConfigNodePath parsePathNodeExpression(Iterator expression, - ConfigOrigin origin, String originalText, ConfigSyntax flavor) { - ArrayList pathTokens = new ArrayList<>(); - Path path = parsePathExpression(expression, origin, originalText, pathTokens, flavor); - return new ConfigNodePath(path, pathTokens); - } - - // originalText may be null if not available - protected static Path parsePathExpression(Iterator expression, - ConfigOrigin origin, String originalText, - ArrayList pathTokens, - ConfigSyntax flavor) { - // each builder in "buf" is an element in the path. - List buf = new ArrayList<>(); - buf.add(new Element("", false)); - - if (!expression.hasNext()) { - throw new ConfigException.BadPath(origin, originalText, - "Expecting a field name or path here, but got nothing"); - } - - while (expression.hasNext()) { - Token t = expression.next(); - - if (pathTokens != null) { - pathTokens.add(t); - } - - // Ignore all IgnoredWhitespace tokens - if (Tokens.isIgnoredWhitespace(t)) { - continue; - } - - if (Tokens.isValueWithType(t, ConfigValueType.STRING)) { - AbstractConfigValue v = Tokens.getValue(t); - // this is a quoted string; so any periods - // in here don't count as path separators - String s = v.transformToString(); - - addPathText(buf, true, s); - } else if (t == Tokens.END) { - // ignore this; when parsing a file, it should not happen - // since we're parsing a token list rather than the main - // token iterator, and when parsing a path expression from the - // API, it's expected to have an END. - } else { - // any periods outside of a quoted string count as - // separators - String text; - if (Tokens.isValue(t)) { - // appending a number here may add - // a period, but we _do_ count those as path - // separators, because we basically want - // "foo 3.0bar" to parse as a string even - // though there's a number in it. The fact that - // we tokenize non-string values is largely an - // implementation detail. - AbstractConfigValue v = Tokens.getValue(t); - - // We need to split the tokens on a . so that we can get sub-paths but still preserve - // the original path text when doing an insertion - if (pathTokens != null) { - pathTokens.remove(pathTokens.size() - 1); - pathTokens.addAll(splitTokenOnPeriod(t, flavor)); - } - text = v.transformToString(); - } else if (Tokens.isUnquotedText(t)) { - // We need to split the tokens on a . so that we can get sub-paths but still preserve - // the original path text when doing an insertion on ConfigNodeObjects - if (pathTokens != null) { - pathTokens.remove(pathTokens.size() - 1); - pathTokens.addAll(splitTokenOnPeriod(t, flavor)); - } - text = Tokens.getUnquotedText(t); - } else { - throw new ConfigException.BadPath( - origin, - originalText, - "Token not allowed in path expression: " - + t - + " (you can double-quote this token if you really want it here)"); - } - - addPathText(buf, false, text); - } - } - - PathBuilder pb = new PathBuilder(); - for (Element e : buf) { - if (e.sb.length() == 0 && !e.canBeEmpty) { - throw new ConfigException.BadPath( - origin, - originalText, - "path has a leading, trailing, or two adjacent period '.' (use quoted \"\" empty string if you want an empty element)"); - } else { - pb.appendKey(e.sb.toString()); - } - } - - return pb.result(); - } - - private static Collection splitTokenOnPeriod(Token t, ConfigSyntax flavor) { - - String tokenText = t.tokenText(); - if (tokenText.equals(ConfigParseOptions.PATH_TOKEN_SEPARATOR)) { - return Collections.singletonList(t); - } - String[] splitToken = tokenText.split(ConfigParseOptions.PATH_TOKEN_SEPARATOR); - ArrayList splitTokens = new ArrayList<>(); - for (String s : splitToken) { - if (flavor == ConfigSyntax.CONF) { - splitTokens.add(Tokens.newUnquotedText(t.origin(), s)); - } else { - splitTokens.add(Tokens.newString(t.origin(), s, "\"" + s + "\"")); - } - splitTokens.add(Tokens.newUnquotedText(t.origin(), ConfigParseOptions.PATH_TOKEN_SEPARATOR)); - } - - if (!tokenText.startsWith(ConfigParseOptions.PATH_TOKEN_SEPARATOR, tokenText.length() - ConfigParseOptions.PATH_TOKEN_SEPARATOR.length())) { - splitTokens.remove(splitTokens.size() - 1); - } - - return splitTokens; - } - - private static void addPathText(List buf, boolean wasQuoted, - String newText) { - - int i = wasQuoted ? -1 : newText.indexOf(ConfigParseOptions.PATH_TOKEN_SEPARATOR); - Element current = buf.get(buf.size() - 1); - if (i < 0) { - // add to current path element - current.sb.append(newText); - // any empty quoted string means this element can - // now be empty. - if (wasQuoted && current.sb.length() == 0) { - current.canBeEmpty = true; - } - } else { - // "buf" plus up to the period is an element - current.sb.append(newText, 0, i); - // then start a new element - buf.add(new Element("", false)); - // recurse to consume remainder of newText - addPathText(buf, false, newText.substring(i + ConfigParseOptions.PATH_TOKEN_SEPARATOR.length())); - } - } - - // the idea is to see if the string has any chars or features - // that might require the full parser to deal with. - private static boolean looksUnsafeForFastParser(String s) { - // TODO: maybe we should rewrite this function using ConfigParseOptions.pathTokenSeparator - boolean lastWasDot = true; // start of path is also a "dot" - int len = s.length(); - if (s.isEmpty()) { - return true; - } - if (s.charAt(0) == '.') { - return true; - } - if (s.charAt(len - 1) == '.') { - return true; - } - - for (int i = 0; i < len; ++i) { - char c = s.charAt(i); - if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { - lastWasDot = false; - } else if (c == '.') { - if (lastWasDot) { - return true; // ".." means we need to throw an error - } - lastWasDot = true; - } else if (c == '-') { - if (lastWasDot) { - return true; - } - } else { - return true; - } - } - - if (lastWasDot) { - return true; - } - - return false; - } - - private static Path fastPathBuild(Path tail, String s, int end) { - - // lastIndexOf takes last index it should look at, end - 1 not end - int splitAt = s.lastIndexOf(ConfigParseOptions.PATH_TOKEN_SEPARATOR, end - 1); - ArrayList tokens = new ArrayList<>(); - tokens.add(Tokens.newUnquotedText(null, s)); - // this works even if splitAt is -1; then we start the substring at 0 - - if (splitAt < 0) { - Path withOneMoreElement = new Path(s.substring(0, end), tail); - return withOneMoreElement; - } else { - Path withOneMoreElement = new Path(s.substring(splitAt + ConfigParseOptions.PATH_TOKEN_SEPARATOR.length(), end), tail); - return fastPathBuild(withOneMoreElement, s, splitAt); - } - } - - // do something much faster than the full parser if - // we just have something like "foo" or "foo.bar" - private static Path speculativeFastParsePath(String path) { - String s = ConfigImplUtil.unicodeTrim(path); - if (looksUnsafeForFastParser(s)) { - return null; - } - - return fastPathBuild(null, s, s.length()); - } - - static class Element { - StringBuilder sb; - // an element can be empty if it has a quoted empty string "" in it - boolean canBeEmpty; - - Element(String initial, boolean canBeEmpty) { - this.canBeEmpty = canBeEmpty; - this.sb = new StringBuilder(initial); - } - - @Override - public String toString() { - return "Element(" + sb.toString() + "," + canBeEmpty + ")"; - } - } -} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/SimpleConfigObject.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/SimpleConfigObject.java deleted file mode 100644 index ed52f4b4105..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/SimpleConfigObject.java +++ /dev/null @@ -1,568 +0,0 @@ -/* - * Copyright (C) 2011-2012 Typesafe Inc. - */ - -package org.apache.seatunnel.shade.com.typesafe.config.impl; - -import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigOrigin; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; - -import java.io.ObjectStreamException; -import java.io.Serializable; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -final class SimpleConfigObject extends AbstractConfigObject implements Serializable { - private static final long serialVersionUID = 2L; - private final Map value; - private final boolean resolved; - private final boolean ignoresFallbacks; - private static final SimpleConfigObject EMPTY_INSTANCE = empty(SimpleConfigOrigin.newSimple("empty config")); - private static final int HASH_CODE = 41; - - SimpleConfigObject(ConfigOrigin origin, Map value, ResolveStatus status, boolean ignoresFallbacks) { - super(origin); - if (value == null) { - throw new ConfigException.BugOrBroken("creating config object with null map"); - } else { - this.value = value; - this.resolved = status == ResolveStatus.RESOLVED; - this.ignoresFallbacks = ignoresFallbacks; - if (status != ResolveStatus.fromValues(value.values())) { - throw new ConfigException.BugOrBroken("Wrong resolved status on " + this); - } - } - } - - SimpleConfigObject(ConfigOrigin origin, Map value) { - this(origin, value, ResolveStatus.fromValues(value.values()), false); - } - - public SimpleConfigObject withOnlyKey(String key) { - return this.withOnlyPath(Path.newKey(key)); - } - - public SimpleConfigObject withoutKey(String key) { - return this.withoutPath(Path.newKey(key)); - } - - protected SimpleConfigObject withOnlyPathOrNull(Path path) { - String key = path.first(); - Path next = path.remainder(); - AbstractConfigValue v = this.value.get(key); - if (next != null) { - if (v instanceof AbstractConfigObject) { - v = ((AbstractConfigObject) v).withOnlyPathOrNull(next); - } else { - v = null; - } - } - - return v == null ? null : new SimpleConfigObject(this.origin(), Collections.singletonMap(key, v), v.resolveStatus(), this.ignoresFallbacks); - } - - SimpleConfigObject withOnlyPath(Path path) { - SimpleConfigObject o = this.withOnlyPathOrNull(path); - return o == null ? new SimpleConfigObject(this.origin(), Collections.emptyMap(), ResolveStatus.RESOLVED, this.ignoresFallbacks) : o; - } - - SimpleConfigObject withoutPath(Path path) { - String key = path.first(); - Path next = path.remainder(); - AbstractConfigValue v = this.value.get(key); - HashMap smaller; - if (next != null && v instanceof AbstractConfigObject) { - v = ((AbstractConfigObject) v).withoutPath(next); - smaller = new HashMap<>(this.value); - smaller.put(key, v); - return new SimpleConfigObject(this.origin(), smaller, ResolveStatus.fromValues(smaller.values()), this.ignoresFallbacks); - } else if (next == null && v != null) { - smaller = new HashMap<>(this.value.size() - 1); - - for (Entry stringAbstractConfigValueEntry : this.value.entrySet()) { - if (!stringAbstractConfigValueEntry.getKey().equals(key)) { - smaller.put(stringAbstractConfigValueEntry.getKey(), stringAbstractConfigValueEntry.getValue()); - } - } - - return new SimpleConfigObject(this.origin(), smaller, ResolveStatus.fromValues(smaller.values()), this.ignoresFallbacks); - } else { - return this; - } - } - - public SimpleConfigObject withValue(String key, ConfigValue v) { - if (v == null) { - throw new ConfigException.BugOrBroken("Trying to store null ConfigValue in a ConfigObject"); - } else { - Map newMap; - if (this.value.isEmpty()) { - newMap = Collections.singletonMap(key, (AbstractConfigValue) v); - } else { - newMap = new HashMap<>(this.value); - newMap.put(key, v); - } - - return new SimpleConfigObject(this.origin(), newMap, ResolveStatus.fromValues(newMap.values()), this.ignoresFallbacks); - } - } - - SimpleConfigObject withValue(Path path, ConfigValue v) { - String key = path.first(); - Path next = path.remainder(); - if (next == null) { - return this.withValue(key, v); - } else { - AbstractConfigValue child = this.value.get(key); - if (child instanceof AbstractConfigObject) { - return this.withValue(key, ((AbstractConfigObject) child).withValue(next, v)); - } else { - SimpleConfig subtree = ((AbstractConfigValue) v).atPath(SimpleConfigOrigin.newSimple("withValue(" + next.render() + ")"), next); - return this.withValue(key, subtree.root()); - } - } - } - - protected AbstractConfigValue attemptPeekWithPartialResolve(String key) { - return this.value.get(key); - } - - private SimpleConfigObject newCopy(ResolveStatus newStatus, ConfigOrigin newOrigin, boolean newIgnoresFallbacks) { - return new SimpleConfigObject(newOrigin, this.value, newStatus, newIgnoresFallbacks); - } - - protected SimpleConfigObject newCopy(ResolveStatus newStatus, ConfigOrigin newOrigin) { - return this.newCopy(newStatus, newOrigin, this.ignoresFallbacks); - } - - protected SimpleConfigObject withFallbacksIgnored() { - return this.ignoresFallbacks ? this : this.newCopy(this.resolveStatus(), this.origin(), true); - } - - ResolveStatus resolveStatus() { - return ResolveStatus.fromBoolean(this.resolved); - } - - public SimpleConfigObject replaceChild(AbstractConfigValue child, AbstractConfigValue replacement) { - Map newChildren = new HashMap<>(this.value); - Iterator> var4 = newChildren.entrySet().iterator(); - - Entry old; - do { - if (!var4.hasNext()) { - throw new ConfigException.BugOrBroken("SimpleConfigObject.replaceChild did not find " + child + " in " + this); - } - - old = var4.next(); - } while (old.getValue() != child); - - if (replacement != null) { - old.setValue(replacement); - } else { - newChildren.remove(old.getKey()); - } - - return new SimpleConfigObject(this.origin(), newChildren, ResolveStatus.fromValues(newChildren.values()), this.ignoresFallbacks); - } - - public boolean hasDescendant(AbstractConfigValue descendant) { - Iterator var2 = this.value.values().iterator(); - - AbstractConfigValue child; - do { - if (!var2.hasNext()) { - var2 = this.value.values().iterator(); - - do { - if (!var2.hasNext()) { - return false; - } - - child = var2.next(); - } while (!(child instanceof Container) || !((Container) child).hasDescendant(descendant)); - - return true; - } - - child = var2.next(); - } while (child != descendant); - - return true; - } - - protected boolean ignoresFallbacks() { - return this.ignoresFallbacks; - } - - public Map unwrapped() { - Map m = new HashMap<>(); - - for (Entry stringAbstractConfigValueEntry : this.value.entrySet()) { - m.put(stringAbstractConfigValueEntry.getKey(), stringAbstractConfigValueEntry.getValue().unwrapped()); - } - - return m; - } - - protected SimpleConfigObject mergedWithObject(AbstractConfigObject abstractFallback) { - this.requireNotIgnoringFallbacks(); - if (!(abstractFallback instanceof SimpleConfigObject)) { - throw new ConfigException.BugOrBroken("should not be reached (merging non-SimpleConfigObject)"); - } else { - SimpleConfigObject fallback = (SimpleConfigObject) abstractFallback; - boolean changed = false; - boolean allResolved = true; - Map merged = new HashMap<>(); - Set allKeys = new HashSet<>(); - allKeys.addAll(this.keySet()); - allKeys.addAll(fallback.keySet()); - - for (String key : allKeys) { - AbstractConfigValue first = this.value.get(key); - AbstractConfigValue second = fallback.value.get(key); - AbstractConfigValue kept; - if (first == null) { - kept = second; - } else if (second == null) { - kept = first; - } else { - kept = first.withFallback(second); - } - - merged.put(key, kept); - if (first != kept) { - changed = true; - } - - if (kept.resolveStatus() == ResolveStatus.UNRESOLVED) { - allResolved = false; - } - } - - ResolveStatus newResolveStatus = ResolveStatus.fromBoolean(allResolved); - boolean newIgnoresFallbacks = fallback.ignoresFallbacks(); - if (changed) { - return new SimpleConfigObject(mergeOrigins(this, fallback), merged, newResolveStatus, newIgnoresFallbacks); - } else if (newResolveStatus == this.resolveStatus() && newIgnoresFallbacks == this.ignoresFallbacks()) { - return this; - } else { - return this.newCopy(newResolveStatus, this.origin(), newIgnoresFallbacks); - } - } - } - - private SimpleConfigObject modify(NoExceptionsModifier modifier) { - try { - return this.modifyMayThrow(modifier); - } catch (RuntimeException var3) { - throw var3; - } catch (Exception var4) { - throw new ConfigException.BugOrBroken("unexpected checked exception", var4); - } - } - - private SimpleConfigObject modifyMayThrow(Modifier modifier) throws Exception { - Map changes = null; - - for (String k : this.keySet()) { - AbstractConfigValue v = this.value.get(k); - AbstractConfigValue modified = modifier.modifyChildMayThrow(k, v); - if (modified != v) { - if (changes == null) { - changes = new HashMap<>(); - } - - changes.put(k, modified); - } - } - - if (changes == null) { - return this; - } else { - Map modified = new HashMap<>(); - boolean sawUnresolved = false; - - for (String k : this.keySet()) { - AbstractConfigValue newValue; - if (changes.containsKey(k)) { - newValue = changes.get(k); - if (newValue != null) { - modified.put(k, newValue); - if (newValue.resolveStatus() == ResolveStatus.UNRESOLVED) { - sawUnresolved = true; - } - } - } else { - newValue = this.value.get(k); - modified.put(k, newValue); - if (newValue.resolveStatus() == ResolveStatus.UNRESOLVED) { - sawUnresolved = true; - } - } - } - - return new SimpleConfigObject(this.origin(), modified, sawUnresolved ? ResolveStatus.UNRESOLVED : ResolveStatus.RESOLVED, this.ignoresFallbacks()); - } - } - - ResolveResult resolveSubstitutions(ResolveContext context, ResolveSource source) throws NotPossibleToResolve { - if (this.resolveStatus() == ResolveStatus.RESOLVED) { - return ResolveResult.make(context, this); - } else { - ResolveSource sourceWithParent = source.pushParent(this); - - try { - SimpleConfigObject.ResolveModifier modifier = new SimpleConfigObject.ResolveModifier(context, sourceWithParent); - AbstractConfigValue value = this.modifyMayThrow(modifier); - return ResolveResult.make(modifier.context, value).asObjectResult(); - } catch (NotPossibleToResolve | RuntimeException var6) { - throw var6; - } catch (Exception var8) { - throw new ConfigException.BugOrBroken("unexpected checked exception", var8); - } - } - } - - SimpleConfigObject relativized(final Path prefix) { - return this.modify(new NoExceptionsModifier() { - public AbstractConfigValue modifyChild(String key, AbstractConfigValue v) { - return v.relativized(prefix); - } - }); - } - - protected void render(StringBuilder sb, int indent, boolean atRoot, ConfigRenderOptions options) { - if (this.isEmpty()) { - sb.append("{}"); - } else { - boolean outerBraces = options.getJson() || !atRoot; - int innerIndent; - if (outerBraces) { - innerIndent = indent + 1; - sb.append("{"); - if (options.getFormatted()) { - sb.append('\n'); - } - } else { - innerIndent = indent; - } - - int separatorCount = 0; - String[] keys = this.keySet().toArray(new String[0]); - - for (String k : keys) { - AbstractConfigValue v = this.value.get(k); - if (options.getOriginComments()) { - String[] lines = v.origin().description().split("\n"); - - for (String l : lines) { - indent(sb, indent + 1, options); - sb.append('#'); - if (!l.isEmpty()) { - sb.append(' '); - } - - sb.append(l); - sb.append("\n"); - } - } - - if (options.getComments()) { - - for (String comment : v.origin().comments()) { - indent(sb, innerIndent, options); - sb.append("#"); - if (!comment.startsWith(" ")) { - sb.append(' '); - } - - sb.append(comment); - sb.append("\n"); - } - } - - indent(sb, innerIndent, options); - v.render(sb, innerIndent, false, k, options); - if (options.getFormatted()) { - if (options.getJson()) { - sb.append(","); - separatorCount = 2; - } else { - separatorCount = 1; - } - - sb.append('\n'); - } else { - sb.append(","); - separatorCount = 1; - } - } - - sb.setLength(sb.length() - separatorCount); - if (outerBraces) { - if (options.getFormatted()) { - sb.append('\n'); - indent(sb, indent, options); - } - - sb.append("}"); - } - } - - if (atRoot && options.getFormatted()) { - sb.append('\n'); - } - - } - - public AbstractConfigValue get(Object key) { - return this.value.get(key); - } - - private static boolean mapEquals(Map a, Map b) { - if (a == b) { - return true; - } else { - Set aKeys = a.keySet(); - Set bKeys = b.keySet(); - if (aKeys.equals(bKeys)) { - Iterator var4 = aKeys.iterator(); - - String key; - do { - if (!var4.hasNext()) { - return true; - } - - key = var4.next(); - } while (a.get(key).equals(b.get(key))); - - } - return false; - } - } - - @SuppressWarnings("magicnumber") - private static int mapHash(Map m) { - List keys = new ArrayList<>(m.keySet()); - Collections.sort(keys); - int valuesHash = 0; - - String k; - for (Iterator var3 = keys.iterator(); var3.hasNext(); valuesHash += m.get(k).hashCode()) { - k = var3.next(); - } - - return HASH_CODE * (HASH_CODE + keys.hashCode()) + valuesHash; - } - - protected boolean canEqual(Object other) { - return other instanceof ConfigObject; - } - - public boolean equals(Object other) { - if (!(other instanceof ConfigObject)) { - return false; - } else { - return this.canEqual(other) && mapEquals(this, (ConfigObject) other); - } - } - - public int hashCode() { - return mapHash(this); - } - - public boolean containsKey(Object key) { - return this.value.containsKey(key); - } - - public Set keySet() { - return this.value.keySet(); - } - - public boolean containsValue(Object v) { - return this.value.containsValue(v); - } - - public Set> entrySet() { - HashSet> entries = new HashSet<>(); - - for (Entry stringAbstractConfigValueEntry : this.value.entrySet()) { - entries.add(new AbstractMap.SimpleImmutableEntry<>(stringAbstractConfigValueEntry.getKey(), stringAbstractConfigValueEntry.getValue())); - } - - return entries; - } - - public boolean isEmpty() { - return this.value.isEmpty(); - } - - public int size() { - return this.value.size(); - } - - public Collection values() { - return new HashSet<>(this.value.values()); - } - - static SimpleConfigObject empty() { - return EMPTY_INSTANCE; - } - - static SimpleConfigObject empty(ConfigOrigin origin) { - return origin == null ? empty() : new SimpleConfigObject(origin, Collections.emptyMap()); - } - - static SimpleConfigObject emptyMissing(ConfigOrigin baseOrigin) { - return new SimpleConfigObject(SimpleConfigOrigin.newSimple(baseOrigin.description() + " (not found)"), Collections.emptyMap()); - } - - private Object writeReplace() throws ObjectStreamException { - return new SerializedConfigValue(this); - } - - private static final class ResolveModifier implements Modifier { - final Path originalRestrict; - ResolveContext context; - final ResolveSource source; - - ResolveModifier(ResolveContext context, ResolveSource source) { - this.context = context; - this.source = source; - this.originalRestrict = context.restrictToChild(); - } - - public AbstractConfigValue modifyChildMayThrow(String key, AbstractConfigValue v) throws NotPossibleToResolve { - if (this.context.isRestrictedToChild()) { - if (key.equals(this.context.restrictToChild().first())) { - Path remainder = this.context.restrictToChild().remainder(); - if (remainder != null) { - ResolveResult result = this.context.restrict(remainder).resolve(v, this.source); - this.context = result.context.unrestricted().restrict(this.originalRestrict); - return result.value; - } else { - return v; - } - } else { - return v; - } - } else { - ResolveResult result = this.context.unrestricted().resolve(v, this.source); - this.context = result.context.unrestricted().restrict(this.originalRestrict); - return result.value; - } - } - } -} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/CompleteTest.java b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/CompleteTest.java deleted file mode 100644 index 1652b5d45c1..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/CompleteTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.config; - -import org.apache.seatunnel.config.utils.FileUtils; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.Map; - -public class CompleteTest { - - @Test - public void testVariables() throws URISyntaxException { - // We use a map to mock the system property, since the system property will be only loaded once - // after the test is run. see Issue #1670 - Map systemProperties = new HashMap<>(); - systemProperties.put("dt", "20190318"); - systemProperties.put("city2", "shanghai"); - - Config config = ConfigFactory - .parseFile(FileUtils.getFileFromResources("/seatunnel/variables.conf")) - .resolveWith(ConfigFactory.parseMap(systemProperties), ConfigResolveOptions.defaults().setAllowUnresolved(true)); - String sql1 = config.getConfigList("transform").get(1).getString("sql"); - String sql2 = config.getConfigList("transform").get(2).getString("sql"); - - Assertions.assertTrue(sql1.contains("shanghai")); - Assertions.assertTrue(sql2.contains("20190318")); - - } - -} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java deleted file mode 100644 index 4f24ab7afde..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.config; - -import org.apache.seatunnel.config.utils.FileUtils; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.List; - -public class ConfigFactoryTest { - - @Test - public void testBasicParseAppConf() throws URISyntaxException { - - Config config = ConfigFactory.parseFile(FileUtils.getFileFromResources("/factory/config.conf")); - - Assertions.assertTrue(config.hasPath("env")); - Assertions.assertTrue(config.hasPath("source")); - Assertions.assertTrue(config.hasPath("transform")); - Assertions.assertTrue(config.hasPath("sink")); - - // check evn config - Config env = config.getConfig("env"); - Assertions.assertEquals("SeaTunnel", env.getString("spark.app.name")); - Assertions.assertEquals("2", env.getString("spark.executor.instances")); - Assertions.assertEquals("1", env.getString("spark.executor.cores")); - Assertions.assertEquals("1g", env.getString("spark.executor.memory")); - Assertions.assertEquals("5", env.getString("spark.stream.batchDuration")); - - // check custom plugin - Assertions.assertEquals("c.Console", config.getConfigList("sink").get(1).getString("plugin_name")); - } - - @Test - public void testTransformOrder() throws URISyntaxException { - - Config config = ConfigFactory.parseFile(FileUtils.getFileFromResources("/factory/config.conf")); - - String[] pluginNames = {"split", "sql1", "sql2", "sql3", "json"}; - - List transforms = config.getConfigList("transform"); - Assertions.assertEquals(pluginNames.length, transforms.size()); - - for (int i = 0; i < transforms.size(); i++) { - String parsedPluginName = String.valueOf(transforms.get(i).root().get("plugin_name").unwrapped()); - Assertions.assertEquals(pluginNames[i], parsedPluginName); - } - - } - - @Test - public void testQuotedString() throws URISyntaxException { - List keys = Arrays.asList("spark.app.name", "spark.executor.instances", "spark.executor.cores", - "spark.executor.memory", "spark.stream.batchDuration"); - - Config config = ConfigFactory.parseFile(FileUtils.getFileFromResources("/factory/config.conf")); - Config evnConfig = config.getConfig("env"); - evnConfig.entrySet().forEach(entry -> Assertions.assertTrue(keys.contains(entry.getKey()))); - - } -} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/JsonFormatTest.java b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/JsonFormatTest.java deleted file mode 100644 index bc39f5cd416..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/JsonFormatTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.config; - -import org.apache.seatunnel.config.utils.FileUtils; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.net.URISyntaxException; - -public class JsonFormatTest { - - @Test - public void testJsonFormat() throws URISyntaxException { - - Config json = ConfigFactory - .parseFile(FileUtils.getFileFromResources("/json/spark.batch.json")) - .resolveWith(ConfigFactory.systemProperties(), - ConfigResolveOptions.defaults().setAllowUnresolved(true)); - - Config config = ConfigFactory - .parseFile(FileUtils.getFileFromResources("/json/spark.batch.conf")) - .resolveWith(ConfigFactory.systemProperties(), - ConfigResolveOptions.defaults().setAllowUnresolved(true)); - - Assertions.assertEquals(config.atPath("transform"), json.atPath("transform")); - Assertions.assertEquals(config.atPath("sink"), json.atPath("sink")); - Assertions.assertEquals(config.atPath("source"), json.atPath("source")); - Assertions.assertEquals(config.atPath("env"), json.atPath("env")); - - } - -} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/utils/FileUtils.java b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/utils/FileUtils.java deleted file mode 100644 index 3b0e01b79f9..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/utils/FileUtils.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.config.utils; - -import java.io.File; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Paths; - -public final class FileUtils { - - private FileUtils() { - } - - // get file from classpath, resources folder - public static File getFileFromResources(String fileName) throws URISyntaxException { - URL resource = FileUtils.class.getResource(fileName); - if (resource == null) { - throw new IllegalArgumentException("file is not found!"); - } - return Paths.get(resource.toURI()).toFile(); - } - -} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf b/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf deleted file mode 100644 index af8c82b8ffa..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf +++ /dev/null @@ -1,66 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -###### -###### This config file is a demonstration of batch processing in seatunnel config -###### - -env { - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - "spark.executor.cores" = 1 - "spark.executor.memory" = "1g" - "spark.stream.batchDuration" = 5 -} - -source { - - fakeStream { - content = ["Hello World, SeaTunnel"] - } - -} - -transform { - - split { - fields = ["msg", "name"] - delimiter = "," - } - - sql1 { - sql = "sql1" - } - - sql2 { - sql = "sql2" - } - - sql3 { - sql = "sql3" - } - - json { - sql = "sql3" - } - -} - -sink { - Console {} - c.Console {} -} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.conf b/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.conf deleted file mode 100644 index 27ad42b4139..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.conf +++ /dev/null @@ -1,72 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -###### -###### This config file is a demonstration of batch processing in SeaTunnel config -###### - -env { - # You can set spark configuration here - # see available properties defined by spark: https://spark.apache.org/docs/latest/configuration.html#available-properties - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" -} - -source { - # This is a example input plugin **only for test and demonstrate the feature input plugin** - Fake { - result_table_name = "my_dataset" - } - - # You can also use other input plugins, such as hdfs - # hdfs { - # result_table_name = "accesslog" - # path = "hdfs://hadoop-cluster-01/nginx/accesslog" - # format = "json" - # } - - # If you would like to get more information about how to configure seatunnel and see full list of input plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/source-plugins/Fake -} - -transform { - # split data by specific delimiter - - # you can also use other transform plugins, such as sql - # sql { - # sql = "select * from accesslog where request_time > 1000" - # } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/transform-plugins/Split -} - -sink { - # choose stdout output plugin to output data to console - Console {} - - # you can also you other output plugins, such as sql - # hdfs { - # path = "hdfs://hadoop-cluster-01/nginx/accesslog_processed" - # save_mode = "append" - # } - - # If you would like to get more information about how to configure seatunnel and see full list of output plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/sink-plugins/Console -} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.json b/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.json deleted file mode 100644 index f0f68aee576..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "env" : { - "spark.app.name" : "SeaTunnel", - "spark.executor.cores" : 1, - "spark.executor.instances" : 2, - "spark.executor.memory" : "1g" - }, - "sink" : [ - { - "plugin_name" : "Console" - } - ], - "source" : [ - { - "plugin_name" : "Fake", - "result_table_name" : "my_dataset" - } - ], - "transform" : [] -} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf b/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf deleted file mode 100644 index f06fb1c1619..00000000000 --- a/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -spark { - spark.stream.batchDuration = 5 - - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" -} - -source { - fakestream { - content = [ - "20190318, beijing, first message", - "20190319, shanghai, second message", - "20190318, shanghai, third message" - ] - rate = 1 - } -} - -transform { - split { - fields = ["dt", "city", "msg"] - delimiter = "," - } - - sql { - table_name = "user_view" - sql = "select * from user_view where city = '"${city2}"'" - result_table_name = "result1" - } - - sql { - table_name = "user_view" - sql = "select * from user_view where dt = '"${dt}"'" - result_table_name = "result2" - } -} - -sink { - stdout { - source_table_name="result1" - } - - stdout { - } -} diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java similarity index 76% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index 12e1fb922b8..644de5d1f6b 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -1,16 +1,19 @@ -package org.apache.seatunnel.e2e.spark.v2.jdbc; +package org.apache.seatunnel.connectors.seatunnel.jdbc; import static org.testcontainers.shaded.org.awaitility.Awaitility.given; -import org.apache.seatunnel.e2e.spark.SparkContainer; +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.TestContainer; import com.google.common.collect.Lists; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestTemplate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.Container; @@ -30,7 +33,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; -public class JdbcDb2IT extends SparkContainer { +public class JdbcDb2IT extends TestSuiteBase implements TestResource { private static final Logger LOG = LoggerFactory.getLogger(JdbcDb2IT.class); /** @@ -39,7 +42,7 @@ public class JdbcDb2IT extends SparkContainer { private static final String IMAGE = "ibmcom/db2:latest"; private static final String HOST = "spark_e2e_db2"; private static final int PORT = 50000; - private static final int LOCAL_PORT = 50001; + private static final int LOCAL_PORT = 50000; private static final String USER = "DB2INST1"; private static final String PASSWORD = "123456"; private static final String DRIVER = "com.ibm.db2.jcc.DB2Driver"; @@ -48,13 +51,14 @@ public class JdbcDb2IT extends SparkContainer { private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE"; private static final String SINK_TABLE = "E2E_TABLE_SINK"; private String JDBC_URL; - private GenericContainer server; + private GenericContainer dbserver; private Connection jdbcConnection; - @BeforeEach - public void startDB2Container() throws ClassNotFoundException, SQLException { - server = new GenericContainer<>(IMAGE) - .withNetwork(NETWORK) + @BeforeAll + @Override + public void startUp() throws Exception { + dbserver = new GenericContainer<>(IMAGE) + .withNetwork(TestContainer.NETWORK) .withNetworkAliases(HOST) .withPrivilegedMode(true) .withLogConsumer(new Slf4jLogConsumer(LOG)) @@ -62,9 +66,9 @@ public void startDB2Container() throws ClassNotFoundException, SQLException { .withEnv("DBNAME", DATABASE) .withEnv("LICENSE", "accept") ; - server.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); - Startables.deepStart(Stream.of(server)).join(); - JDBC_URL = String.format("jdbc:db2://%s:%s/%s", server.getContainerIpAddress(), LOCAL_PORT, DATABASE); + dbserver.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); + Startables.deepStart(Stream.of(dbserver)).join(); + JDBC_URL = String.format("jdbc:db2://%s:%s/%s", dbserver.getHost(), LOCAL_PORT, DATABASE); LOG.info("DB2 container started"); given().ignoreExceptions() .await() @@ -73,13 +77,13 @@ public void startDB2Container() throws ClassNotFoundException, SQLException { initializeJdbcTable(); } - @AfterEach - public void closeGreenplumContainer() throws SQLException { + @Override + public void tearDown() throws Exception { if (jdbcConnection != null) { jdbcConnection.close(); } - if (server != null) { - server.close(); + if (dbserver != null) { + dbserver.close(); } } @@ -100,7 +104,7 @@ private void initializeJdbcConnection() throws SQLException, ClassNotFoundExcept * init the table */ private void initializeJdbcTable() { - URL resource = JdbcDb2IT.class.getResource("/jdbc/init_sql/db2_init.conf"); + URL resource = JdbcDb2IT.class.getResource("/jdbc/init/db2_init.conf"); if (resource == null) { throw new IllegalArgumentException("can't find find file"); } @@ -138,10 +142,12 @@ void pullImageOK() throws SQLException { assertHasData(SOURCE_TABLE); } - @Test - public void testJdbcSourceAndSink() throws IOException, InterruptedException, SQLException { + + @TestTemplate + @DisplayName("JDBC-DM end to end test") + public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException, SQLException { assertHasData(SOURCE_TABLE); - Container.ExecResult execResult = executeSeaTunnelSparkJob("/jdbc/jdbc_db2_source_and_sink.conf"); + Container.ExecResult execResult = container.executeJob("/jdbc/jdbc_db2_source_and_sink.conf"); Assertions.assertEquals(0, execResult.getExitCode()); assertHasData(SINK_TABLE); } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/init_sql/db2_init.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/init/db2_init.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/init_sql/db2_init.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/init/db2_init.conf diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/jdbc/JdbcDb2IT.java deleted file mode 100644 index 227fd9235d7..00000000000 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/jdbc/JdbcDb2IT.java +++ /dev/null @@ -1,142 +0,0 @@ -package org.apache.seatunnel.e2e.flink.v2.jdbc; - -import static org.testcontainers.shaded.org.awaitility.Awaitility.given; - -import org.apache.seatunnel.e2e.flink.FlinkContainer; - -import com.google.common.collect.Lists; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.Container; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.lifecycle.Startables; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - -public class JdbcDb2IT extends FlinkContainer { - - private static final Logger LOG = LoggerFactory.getLogger(JdbcDb2IT.class); - /** - * db2 in dockerhub - */ - private static final String IMAGE = "ibmcom/db2:latest"; - private static final String HOST = "spark_e2e_db2"; - private static final int PORT = 50000; - private static final String LOCAL_HOST = "localhost"; - private static final int LOCAL_PORT = 50000; - private static final String USER = "DB2INST1"; - private static final String PASSWORD = "123456"; - private static final String DRIVER = "com.ibm.db2.jcc.DB2Driver"; - private static final String JDBC_URL = String.format("jdbc:db2://%s:%s/testdb", LOCAL_HOST, LOCAL_PORT); - - private static final String SOURCE_TABLE = "e2e_table_source"; - private static final String SINK_TABLE = "e2e_table_sink"; - - private GenericContainer server; - private Connection jdbcConnection; - - @BeforeEach - public void startDB2Container() throws ClassNotFoundException { - server = new GenericContainer<>(IMAGE) - .withNetwork(NETWORK) - .withNetworkAliases(HOST) - .withPrivilegedMode(true) - .withLogConsumer(new Slf4jLogConsumer(LOG)) - .withEnv("DB2INST1_PASSWORD", "123456") - .withEnv("DBNAME", "testdb") - .withEnv("LICENSE", "accept") - ; - server.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); - Startables.deepStart(Stream.of(server)).join(); - LOG.info("DB2 container started"); - Class.forName(DRIVER); - given().ignoreExceptions() - .await() - .atMost(180, TimeUnit.SECONDS) - .untilAsserted(this::initializeJdbcConnection); - initializeJdbcTable(); - LOG.info("db2 init success"); - } - - @AfterEach - public void closeGreenplumContainer() throws SQLException { - if (jdbcConnection != null) { - jdbcConnection.close(); - } - if (server != null) { - server.close(); - } - } - - private void initializeJdbcConnection() throws SQLException { - jdbcConnection = DriverManager.getConnection(JDBC_URL, USER, PASSWORD); - } - - /** - * init the table - */ - private void initializeJdbcTable() { - URL resource = JdbcDb2IT.class.getResource("/jdbc/init_sql/db2_init.conf"); - if (resource == null) { - throw new IllegalArgumentException("can't find find file"); - } - String file = resource.getFile(); - Config config = ConfigFactory.parseFile(new File(file)); - assert config.hasPath("table_source") && config.hasPath("DML") && config.hasPath("table_sink"); - try (Statement statement = jdbcConnection.createStatement()) { - // source - String sourceTableDDL = config.getString("table_source"); - statement.execute(sourceTableDDL); - LOG.info("source DDL success"); - String insertSQL = config.getString("DML"); - statement.execute(insertSQL); - LOG.info("source DML success"); - // sink - String sinkTableDDL = config.getString("table_sink"); - statement.execute(sinkTableDDL); - LOG.info("sink DDL success"); - } catch (SQLException e) { - throw new RuntimeException("Initializing table failed!", e); - } - } - - private void assertHasData(String table) { - try (Connection connection = DriverManager.getConnection(JDBC_URL, USER, PASSWORD)) { - Statement statement = connection.createStatement(); - String sql = String.format("select * from %s.%s limit 1", USER, table); - ResultSet source = statement.executeQuery(sql); - Assertions.assertTrue(source.next()); - } catch (SQLException e) { - throw new RuntimeException("server image error", e); - } - } - - @Test - void pullImageOK() { - assertHasData(SOURCE_TABLE); - } - - @Test - public void testJdbcSourceAndSink() throws IOException, InterruptedException { - assertHasData(SOURCE_TABLE); - Container.ExecResult execResult = executeSeaTunnelFlinkJob("/jdbc/jdbc_db2_source_and_sink.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - assertHasData(SINK_TABLE); - } -} From 8e1380f5f27581f5633df40685cc96d4e6f3e642 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 16 Oct 2022 14:07:22 +0800 Subject: [PATCH 07/23] fix db2IT --- .../seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index 644de5d1f6b..59f9a7752f7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -50,7 +50,7 @@ public class JdbcDb2IT extends TestSuiteBase implements TestResource { private static final String DATABASE = "testdb"; private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE"; private static final String SINK_TABLE = "E2E_TABLE_SINK"; - private String JDBC_URL; + private String jdbcUrl; private GenericContainer dbserver; private Connection jdbcConnection; @@ -68,7 +68,7 @@ public void startUp() throws Exception { ; dbserver.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); Startables.deepStart(Stream.of(dbserver)).join(); - JDBC_URL = String.format("jdbc:db2://%s:%s/%s", dbserver.getHost(), LOCAL_PORT, DATABASE); + jdbcUrl = String.format("jdbc:db2://%s:%s/%s", dbserver.getHost(), LOCAL_PORT, DATABASE); LOG.info("DB2 container started"); given().ignoreExceptions() .await() @@ -92,7 +92,7 @@ private void initializeJdbcConnection() throws SQLException, ClassNotFoundExcept properties.setProperty("user", USER); properties.setProperty("password", PASSWORD); Driver driver = (Driver) Class.forName(DRIVER).newInstance(); - jdbcConnection = driver.connect(JDBC_URL, properties); + jdbcConnection = driver.connect(jdbcUrl, properties); Statement statement = jdbcConnection.createStatement(); ResultSet resultSet = statement.executeQuery("select 1 from SYSSTAT.TABLES"); Assertions.assertTrue(resultSet.next()); @@ -142,7 +142,6 @@ void pullImageOK() throws SQLException { assertHasData(SOURCE_TABLE); } - @TestTemplate @DisplayName("JDBC-DM end to end test") public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException, SQLException { From b36b9fd05d11ed408e30d513c29a53c9ab6f9c19 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Mon, 17 Oct 2022 00:14:43 +0800 Subject: [PATCH 08/23] reconvert --- seatunnel-config/README.md | 25 + seatunnel-config/pom.xml | 37 ++ .../seatunnel-config-base/pom.xml | 135 +++++ .../seatunnel-config-shade/pom.xml | 82 +++ .../typesafe/config/ConfigParseOptions.java | 250 ++++++++ .../typesafe/config/impl/ConfigNodePath.java | 57 ++ .../typesafe/config/impl/ConfigParser.java | 569 ++++++++++++++++++ .../shade/com/typesafe/config/impl/Path.java | 242 ++++++++ .../com/typesafe/config/impl/PathParser.java | 296 +++++++++ .../config/impl/SimpleConfigObject.java | 568 +++++++++++++++++ .../apache/seatunnel/config/CompleteTest.java | 54 ++ .../seatunnel/config/ConfigFactoryTest.java | 83 +++ .../seatunnel/config/JsonFormatTest.java | 53 ++ .../seatunnel/config/utils/FileUtils.java | 39 ++ .../src/test/resources/factory/config.conf | 66 ++ .../src/test/resources/json/spark.batch.conf | 72 +++ .../src/test/resources/json/spark.batch.json | 20 + .../test/resources/seatunnel/variables.conf | 62 ++ .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 2 +- 19 files changed, 2711 insertions(+), 1 deletion(-) create mode 100644 seatunnel-config/README.md create mode 100644 seatunnel-config/pom.xml create mode 100644 seatunnel-config/seatunnel-config-base/pom.xml create mode 100644 seatunnel-config/seatunnel-config-shade/pom.xml create mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/ConfigParseOptions.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigNodePath.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigParser.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/Path.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PathParser.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/SimpleConfigObject.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/CompleteTest.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/JsonFormatTest.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/utils/FileUtils.java create mode 100644 seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf create mode 100644 seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.conf create mode 100644 seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.json create mode 100644 seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf diff --git a/seatunnel-config/README.md b/seatunnel-config/README.md new file mode 100644 index 00000000000..4932a3f2c9b --- /dev/null +++ b/seatunnel-config/README.md @@ -0,0 +1,25 @@ +# Introduction +The `seatunnel-config` is used to parse `seatunnel.conf` files. This module is based on `com.typesafe.config`, +We have made some enhancement and import our enhancement by using maven shade. Most of the times, you don't need to directly +using this module, since you can receive from maven repository. + +# How to modify the config module +If you want to modify the config module, you can follow the steps below. +1. Open the `seatunnel-config` module. +```xml + +``` +Open the annuotaion in `pom.xml` file, to import the `seatunnel-config` module. +2. Replace the `config-shade` dependency to project. +```xml + + org.apache.seatunnel + seatunnel-config-shade + ${project.version} + +``` +Add `${project.version}` to `seatunnel-config-shade` everywhere you use. \ No newline at end of file diff --git a/seatunnel-config/pom.xml b/seatunnel-config/pom.xml new file mode 100644 index 00000000000..29a8a5e0cb2 --- /dev/null +++ b/seatunnel-config/pom.xml @@ -0,0 +1,37 @@ + + + + + org.apache.seatunnel + seatunnel + ${revision} + + 4.0.0 + seatunnel-config + pom + + seatunnel-config + + + seatunnel-config-shade + seatunnel-config-base + + diff --git a/seatunnel-config/seatunnel-config-base/pom.xml b/seatunnel-config/seatunnel-config-base/pom.xml new file mode 100644 index 00000000000..5401d09c304 --- /dev/null +++ b/seatunnel-config/seatunnel-config-base/pom.xml @@ -0,0 +1,135 @@ + + + + 4.0.0 + + org.apache.seatunnel + seatunnel-config + ${revision} + ../pom.xml + + seatunnel-config-base + seatunnel-config-base + + UTF-8 + ${java.version} + ${java.version} + true + org.apache.seatunnel.shade + + + + + com.typesafe + config + + + + + ${project.artifactId}-${project.version} + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + true + true + true + false + false + + + com.typesafe:config + + ** + + + META-INF/MANIFEST.MF + META-INF/NOTICE + com/typesafe/config/ConfigParseOptions.class + com/typesafe/config/impl/ConfigParser.class + com/typesafe/config/impl/ConfigNodePath.class + com/typesafe/config/impl/PathParser.class + com/typesafe/config/impl/Path.class + com/typesafe/config/impl/SimpleConfigObject.class + + + + + + com.typesafe.config + ${seatunnel.shade.package}.com.typesafe.config + + + + + + + + + + package + + shade + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + compile + package + + attach-artifact + + + + + ${basedir}/target/${project.artifactId}-${project.version}.jar + jar + optional + + + + + + + + + + \ No newline at end of file diff --git a/seatunnel-config/seatunnel-config-shade/pom.xml b/seatunnel-config/seatunnel-config-shade/pom.xml new file mode 100644 index 00000000000..417d6e2de85 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/pom.xml @@ -0,0 +1,82 @@ + + + + + org.apache.seatunnel + seatunnel-config + ${revision} + ../pom.xml + + 4.0.0 + seatunnel-config-shade + + seatunnel-config-shade + + + UTF-8 + ${java.version} + ${java.version} + true + org.apache.seatunnel.shade + + + + org.apache.seatunnel + seatunnel-config-base + ${project.version} + + + org.scala-lang + scala-library + + + + + + ${project.artifactId}-${project.version} + + + + org.codehaus.mojo + build-helper-maven-plugin + + + compile + package + + attach-artifact + + + + + ${basedir}/target/${project.artifactId}-${project.version}.jar + jar + optional + + + + + + + + + + diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/ConfigParseOptions.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/ConfigParseOptions.java new file mode 100644 index 00000000000..b7949670257 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/ConfigParseOptions.java @@ -0,0 +1,250 @@ +/* + * Copyright (C) 2011-2012 Typesafe Inc. + */ + +package org.apache.seatunnel.shade.com.typesafe.config; + +/** + * A set of options related to parsing. + * + *

+ * This object is immutable, so the "setters" return a new object. + * + *

+ * Here is an example of creating a custom {@code ConfigParseOptions}: + * + * 

+ *     ConfigParseOptions options = ConfigParseOptions.defaults()
+ *         .setSyntax(ConfigSyntax.JSON)
+ *         .setAllowMissing(false)
+ *
+ */ +public final class ConfigParseOptions { + + /** + * a.b.c + * a->b->c + */ + public static final String PATH_TOKEN_SEPARATOR = "->"; + + final ConfigSyntax syntax; + final String originDescription; + final boolean allowMissing; + final ConfigIncluder includer; + final ClassLoader classLoader; + + private ConfigParseOptions(ConfigSyntax syntax, String originDescription, boolean allowMissing, + ConfigIncluder includer, ClassLoader classLoader) { + this.syntax = syntax; + this.originDescription = originDescription; + this.allowMissing = allowMissing; + this.includer = includer; + this.classLoader = classLoader; + } + + /** + * Gets an instance of {@code ConfigParseOptions} with all fields + * set to the default values. Start with this instance and make any + * changes you need. + * + * @return the default parse options + */ + public static ConfigParseOptions defaults() { + return new ConfigParseOptions(null, null, true, null, null); + } + + /** + * Set the file format. If set to null, try to guess from any available + * filename extension; if guessing fails, assume {@link ConfigSyntax#CONF}. + * + * @param syntax a syntax or {@code null} for best guess + * @return options with the syntax set + */ + public ConfigParseOptions setSyntax(ConfigSyntax syntax) { + if (this.syntax == syntax) { + return this; + } else { + return new ConfigParseOptions(syntax, this.originDescription, this.allowMissing, + this.includer, this.classLoader); + } + } + + /** + * Gets the current syntax option, which may be null for "any". + * + * @return the current syntax or null + */ + public ConfigSyntax getSyntax() { + return syntax; + } + + /** + * Set a description for the thing being parsed. In most cases this will be + * set up for you to something like the filename, but if you provide just an + * input stream you might want to improve on it. Set to null to allow the + * library to come up with something automatically. This description is the + * basis for the {@link ConfigOrigin} of the parsed values. + * + * @param originDescription description to put in the {@link ConfigOrigin} + * @return options with the origin description set + */ + public ConfigParseOptions setOriginDescription(String originDescription) { + // findbugs complains about == here but is wrong, do not "fix" + if (this.originDescription == originDescription) { + return this; + } else if (this.originDescription != null && originDescription != null + && this.originDescription.equals(originDescription)) { + return this; + } else { + return new ConfigParseOptions(this.syntax, originDescription, this.allowMissing, + this.includer, this.classLoader); + } + } + + /** + * Gets the current origin description, which may be null for "automatic". + * + * @return the current origin description or null + */ + public String getOriginDescription() { + return originDescription; + } + + /** + * this is package-private, not public API + */ + ConfigParseOptions withFallbackOriginDescription(String originDescription) { + if (this.originDescription == null) { + return setOriginDescription(originDescription); + } else { + return this; + } + } + + /** + * Set to false to throw an exception if the item being parsed (for example + * a file) is missing. Set to true to just return an empty document in that + * case. Note that this setting applies on only to fetching the root document, + * it has no effect on any nested includes. + * + * @param allowMissing true to silently ignore missing item + * @return options with the "allow missing" flag set + */ + public ConfigParseOptions setAllowMissing(boolean allowMissing) { + if (this.allowMissing == allowMissing) { + return this; + } else { + return new ConfigParseOptions(this.syntax, this.originDescription, allowMissing, + this.includer, this.classLoader); + } + } + + /** + * Gets the current "allow missing" flag. + * + * @return whether we allow missing files + */ + public boolean getAllowMissing() { + return allowMissing; + } + + /** + * Set a {@link ConfigIncluder} which customizes how includes are handled. + * null means to use the default includer. + * + * @param includer the includer to use or null for default + * @return new version of the parse options with different includer + */ + public ConfigParseOptions setIncluder(ConfigIncluder includer) { + if (this.includer == includer) { + return this; + } else { + return new ConfigParseOptions(this.syntax, this.originDescription, this.allowMissing, + includer, this.classLoader); + } + } + + /** + * Prepends a {@link ConfigIncluder} which customizes how + * includes are handled. To prepend your includer, the + * library calls {@link ConfigIncluder#withFallback} on your + * includer to append the existing includer to it. + * + * @param includer the includer to prepend (may not be null) + * @return new version of the parse options with different includer + */ + public ConfigParseOptions prependIncluder(ConfigIncluder includer) { + if (includer == null) { + throw new NullPointerException("null includer passed to prependIncluder"); + } + if (this.includer == includer) { + return this; + } else if (this.includer != null) { + return setIncluder(includer.withFallback(this.includer)); + } else { + return setIncluder(includer); + } + } + + /** + * Appends a {@link ConfigIncluder} which customizes how + * includes are handled. To append, the library calls {@link + * ConfigIncluder#withFallback} on the existing includer. + * + * @param includer the includer to append (may not be null) + * @return new version of the parse options with different includer + */ + public ConfigParseOptions appendIncluder(ConfigIncluder includer) { + if (includer == null) { + throw new NullPointerException("null includer passed to appendIncluder"); + } + if (this.includer == includer) { + return this; + } else if (this.includer != null) { + return setIncluder(this.includer.withFallback(includer)); + } else { + return setIncluder(includer); + } + } + + /** + * Gets the current includer (will be null for the default includer). + * + * @return current includer or null + */ + public ConfigIncluder getIncluder() { + return includer; + } + + /** + * Set the class loader. If set to null, + * {@code Thread.currentThread().getContextClassLoader()} will be used. + * + * @param loader a class loader or {@code null} to use thread context class + * loader + * @return options with the class loader set + */ + public ConfigParseOptions setClassLoader(ClassLoader loader) { + if (this.classLoader == loader) { + return this; + } else { + return new ConfigParseOptions(this.syntax, this.originDescription, this.allowMissing, + this.includer, loader); + } + } + + /** + * Get the class loader; never returns {@code null}, if the class loader was + * unset, returns + * {@code Thread.currentThread().getContextClassLoader()}. + * + * @return class loader to use + */ + public ClassLoader getClassLoader() { + if (this.classLoader == null) { + return Thread.currentThread().getContextClassLoader(); + } else { + return this.classLoader; + } + } +} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigNodePath.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigNodePath.java new file mode 100644 index 00000000000..d917d84e849 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigNodePath.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2011-2012 Typesafe Inc. + */ + +package org.apache.seatunnel.shade.com.typesafe.config.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions; + +import java.util.ArrayList; +import java.util.Collection; + +final class ConfigNodePath extends AbstractConfigNode { + private final Path path; + final ArrayList tokens; + + ConfigNodePath(Path path, Collection tokens) { + this.path = path; + this.tokens = new ArrayList<>(tokens); + } + + @Override + protected Collection tokens() { + return tokens; + } + + protected Path value() { + return path; + } + + protected ConfigNodePath subPath(int toRemove) { + int periodCount = 0; + ArrayList tokensCopy = new ArrayList<>(tokens); + for (int i = 0; i < tokensCopy.size(); i++) { + if (Tokens.isUnquotedText(tokensCopy.get(i)) && + tokensCopy.get(i).tokenText().equals(ConfigParseOptions.PATH_TOKEN_SEPARATOR)) { + periodCount++; + } + + if (periodCount == toRemove) { + return new ConfigNodePath(path.subPath(toRemove), tokensCopy.subList(i + 1, tokensCopy.size())); + } + } + throw new ConfigException.BugOrBroken("Tried to remove too many elements from a Path node"); + } + + protected ConfigNodePath first() { + ArrayList tokensCopy = new ArrayList<>(tokens); + for (int i = 0; i < tokensCopy.size(); i++) { + if (Tokens.isUnquotedText(tokensCopy.get(i)) && + tokensCopy.get(i).tokenText().equals(ConfigParseOptions.PATH_TOKEN_SEPARATOR)) { + return new ConfigNodePath(path.subPath(0, 1), tokensCopy.subList(0, i)); + } + } + return this; + } +} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigParser.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigParser.java new file mode 100644 index 00000000000..39b9cb5d463 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/ConfigParser.java @@ -0,0 +1,569 @@ +/* + * Copyright (C) 2011-2012 Typesafe Inc. + */ + +package org.apache.seatunnel.shade.com.typesafe.config.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigIncludeContext; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigOrigin; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigSyntax; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +final class ConfigParser { + static AbstractConfigValue parse(ConfigNodeRoot document, + ConfigOrigin origin, ConfigParseOptions options, + ConfigIncludeContext includeContext) { + ParseContext context = new ParseContext(options.getSyntax(), origin, document, + SimpleIncluder.makeFull(options.getIncluder()), includeContext); + return context.parse(); + } + + private static final class ParseContext { + private int lineNumber; + private final ConfigNodeRoot document; + private final FullIncluder includer; + private final ConfigIncludeContext includeContext; + private final ConfigSyntax flavor; + private final ConfigOrigin baseOrigin; + private final LinkedList pathStack; + + // the number of lists we are inside; this is used to detect the "cannot + // generate a reference to a list element" problem, and once we fix that + // problem we should be able to get rid of this variable. + int arrayCount; + + ParseContext(ConfigSyntax flavor, ConfigOrigin origin, ConfigNodeRoot document, + FullIncluder includer, ConfigIncludeContext includeContext) { + lineNumber = 1; + this.document = document; + this.flavor = flavor; + this.baseOrigin = origin; + this.includer = includer; + this.includeContext = includeContext; + this.pathStack = new LinkedList<>(); + this.arrayCount = 0; + } + + // merge a bunch of adjacent values into one + // value; change unquoted text into a string + // value. + private AbstractConfigValue parseConcatenation(ConfigNodeConcatenation n) { + // this trick is not done in JSON + if (flavor == ConfigSyntax.JSON) { + throw new ConfigException.BugOrBroken("Found a concatenation node in JSON"); + } + + List values = new ArrayList<>(); + + for (AbstractConfigNode node : n.children()) { + AbstractConfigValue v = null; + if (node instanceof AbstractConfigNodeValue) { + v = parseValue((AbstractConfigNodeValue) node, null); + values.add(v); + } + } + + return ConfigConcatenation.concatenate(values); + } + + private SimpleConfigOrigin lineOrigin() { + return ((SimpleConfigOrigin) baseOrigin).withLineNumber(lineNumber); + } + + private ConfigException parseError(String message) { + return parseError(message, null); + } + + private ConfigException parseError(String message, Throwable cause) { + return new ConfigException.Parse(lineOrigin(), message, cause); + } + + private Path fullCurrentPath() { + // pathStack has top of stack at front + if (pathStack.isEmpty()) { + throw new ConfigException.BugOrBroken("Bug in parser; tried to get current path when at root"); + } else { + return new Path(pathStack.descendingIterator()); + } + } + + private AbstractConfigValue parseValue(AbstractConfigNodeValue n, List comments) { + AbstractConfigValue v; + + int startingArrayCount = arrayCount; + + if (n instanceof ConfigNodeSimpleValue) { + v = ((ConfigNodeSimpleValue) n).value(); + } else if (n instanceof ConfigNodeObject) { + + Path path = pathStack.peekFirst(); + + if (path != null && !ConfigSyntax.JSON.equals(flavor) + && ("source".equals(path.first()) + || "transform".equals(path.first()) + || "sink".equals(path.first()))) { + v = parseObjectForSeatunnel((ConfigNodeObject) n); + } else { + v = parseObject((ConfigNodeObject) n); + } + + } else if (n instanceof ConfigNodeArray) { + v = parseArray((ConfigNodeArray) n); + } else if (n instanceof ConfigNodeConcatenation) { + v = parseConcatenation((ConfigNodeConcatenation) n); + } else { + throw parseError("Expecting a value but got wrong node type: " + n.getClass()); + } + + if (comments != null && !comments.isEmpty()) { + v = v.withOrigin(v.origin().prependComments(new ArrayList<>(comments))); + comments.clear(); + } + + if (arrayCount != startingArrayCount) { + throw new ConfigException.BugOrBroken("Bug in config parser: unbalanced array count"); + } + + return v; + } + + private static AbstractConfigObject createValueUnderPath(Path path, + AbstractConfigValue value) { + // for path foo.bar, we are creating + // { "foo" : { "bar" : value } } + List keys = new ArrayList<>(); + + String key = path.first(); + Path remaining = path.remainder(); + while (key != null) { + keys.add(key); + if (remaining == null) { + break; + } else { + key = remaining.first(); + remaining = remaining.remainder(); + } + } + + // the withComments(null) is to ensure comments are only + // on the exact leaf node they apply to. + // a comment before "foo.bar" applies to the full setting + // "foo.bar" not also to "foo" + ListIterator i = keys.listIterator(keys.size()); + String deepest = i.previous(); + AbstractConfigObject o = new SimpleConfigObject(value.origin().withComments(null), + Collections.singletonMap(deepest, value)); + while (i.hasPrevious()) { + Map m = Collections.singletonMap(i.previous(), o); + o = new SimpleConfigObject(value.origin().withComments(null), m); + } + + return o; + } + + private void parseInclude(Map values, ConfigNodeInclude n) { + boolean isRequired = n.isRequired(); + ConfigIncludeContext cic = includeContext.setParseOptions(includeContext.parseOptions().setAllowMissing(!isRequired)); + + AbstractConfigObject obj; + switch (n.kind()) { + case URL: + URL url; + try { + url = new URL(n.name()); + } catch (MalformedURLException e) { + throw parseError("include url() specifies an invalid URL: " + n.name(), e); + } + obj = (AbstractConfigObject) includer.includeURL(cic, url); + break; + + case FILE: + obj = (AbstractConfigObject) includer.includeFile(cic, + new File(n.name())); + break; + + case CLASSPATH: + obj = (AbstractConfigObject) includer.includeResources(cic, n.name()); + break; + + case HEURISTIC: + obj = (AbstractConfigObject) includer + .include(cic, n.name()); + break; + + default: + throw new ConfigException.BugOrBroken("should not be reached"); + } + + // we really should make this work, but for now throwing an + // exception is better than producing an incorrect result. + // See https://github.com/lightbend/config/issues/160 + if (arrayCount > 0 && obj.resolveStatus() != ResolveStatus.RESOLVED) { + throw parseError("Due to current limitations of the config parser, when an include statement is nested inside a list value, " + + "${} substitutions inside the included file cannot be resolved correctly. Either move the include outside of the list value or " + + "remove the ${} statements from the included file."); + } + + if (!pathStack.isEmpty()) { + Path prefix = fullCurrentPath(); + obj = obj.relativized(prefix); + } + + for (String key : obj.keySet()) { + AbstractConfigValue v = obj.get(key); + AbstractConfigValue existing = values.get(key); + if (existing != null) { + values.put(key, v.withFallback(existing)); + } else { + values.put(key, v); + } + } + } + + private SimpleConfigList parseObjectForSeatunnel(ConfigNodeObject n) { + + Map values = new LinkedHashMap<>(); + List valuesList = new ArrayList<>(); + SimpleConfigOrigin objectOrigin = lineOrigin(); + boolean lastWasNewline = false; + + ArrayList nodes = new ArrayList<>(n.children()); + List comments = new ArrayList<>(); + for (int i = 0; i < nodes.size(); i++) { + AbstractConfigNode node = nodes.get(i); + if (node instanceof ConfigNodeComment) { + lastWasNewline = false; + comments.add(((ConfigNodeComment) node).commentText()); + } else if (node instanceof ConfigNodeSingleToken && Tokens.isNewline(((ConfigNodeSingleToken) node).token())) { + lineNumber++; + if (lastWasNewline) { + // Drop all comments if there was a blank line and start a new comment block + comments.clear(); + } + lastWasNewline = true; + } else if (flavor != ConfigSyntax.JSON && node instanceof ConfigNodeInclude) { + parseInclude(values, (ConfigNodeInclude) node); + lastWasNewline = false; + } else if (node instanceof ConfigNodeField) { + lastWasNewline = false; + Path path = ((ConfigNodeField) node).path().value(); + comments.addAll(((ConfigNodeField) node).comments()); + + // path must be on-stack while we parse the value + pathStack.push(path); + if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) { + // we really should make this work, but for now throwing + // an exception is better than producing an incorrect + // result. See + // https://github.com/lightbend/config/issues/160 + if (arrayCount > 0) { + throw parseError("Due to current limitations of the config parser, += does not work nested inside a list. " + + "+= expands to a ${} substitution and the path in ${} cannot currently refer to list elements. " + + "You might be able to move the += outside of the list and then refer to it from inside the list with ${}."); + } + + // because we will put it in an array after the fact so + // we want this to be incremented during the parseValue + // below in order to throw the above exception. + arrayCount += 1; + } + + AbstractConfigNodeValue valueNode; + AbstractConfigValue newValue; + + valueNode = ((ConfigNodeField) node).value(); + + // comments from the key token go to the value token + newValue = parseValue(valueNode, comments); + + if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) { + arrayCount -= 1; + + List concat = new ArrayList<>(2); + AbstractConfigValue previousRef = new ConfigReference(newValue.origin(), + new SubstitutionExpression(fullCurrentPath(), true /* optional */)); + AbstractConfigValue list = new SimpleConfigList(newValue.origin(), + Collections.singletonList(newValue)); + concat.add(previousRef); + concat.add(list); + newValue = ConfigConcatenation.concatenate(concat); + } + + // Grab any trailing comments on the same line + if (i < nodes.size() - 1) { + i++; + while (i < nodes.size()) { + if (nodes.get(i) instanceof ConfigNodeComment) { + ConfigNodeComment comment = (ConfigNodeComment) nodes.get(i); + newValue = newValue.withOrigin(newValue.origin().appendComments( + Collections.singletonList(comment.commentText()))); + break; + } else if (nodes.get(i) instanceof ConfigNodeSingleToken) { + ConfigNodeSingleToken curr = (ConfigNodeSingleToken) nodes.get(i); + if (curr.token() == Tokens.COMMA || Tokens.isIgnoredWhitespace(curr.token())) { + // keep searching, as there could still be a comment + } else { + i--; + break; + } + } else { + i--; + break; + } + i++; + } + } + + pathStack.pop(); + + String key = path.first(); + Path remaining = path.remainder(); + + if (remaining == null) { + + Map m = Collections.singletonMap("plugin_name", key); + newValue = newValue.withFallback(ConfigValueFactory.fromMap(m)); + + values.put(key, newValue); + valuesList.add(newValue); + } else { + if (flavor == ConfigSyntax.JSON) { + throw new ConfigException.BugOrBroken( + "somehow got multi-element path in JSON mode"); + } + + AbstractConfigObject obj = createValueUnderPath( + remaining, newValue); + + Map m = Collections.singletonMap("plugin_name", key); + obj = obj.withFallback(ConfigValueFactory.fromMap(m)); + + values.put(key, obj); + valuesList.add(obj); + } + } + } + + return new SimpleConfigList(objectOrigin, valuesList); + } + + private AbstractConfigObject parseObject(ConfigNodeObject n) { + Map values = new LinkedHashMap<>(); + SimpleConfigOrigin objectOrigin = lineOrigin(); + boolean lastWasNewline = false; + + ArrayList nodes = new ArrayList<>(n.children()); + List comments = new ArrayList<>(); + for (int i = 0; i < nodes.size(); i++) { + AbstractConfigNode node = nodes.get(i); + if (node instanceof ConfigNodeComment) { + lastWasNewline = false; + comments.add(((ConfigNodeComment) node).commentText()); + } else if (node instanceof ConfigNodeSingleToken && Tokens.isNewline(((ConfigNodeSingleToken) node).token())) { + lineNumber++; + if (lastWasNewline) { + // Drop all comments if there was a blank line and start a new comment block + comments.clear(); + } + lastWasNewline = true; + } else if (flavor != ConfigSyntax.JSON && node instanceof ConfigNodeInclude) { + parseInclude(values, (ConfigNodeInclude) node); + lastWasNewline = false; + } else if (node instanceof ConfigNodeField) { + lastWasNewline = false; + Path path = ((ConfigNodeField) node).path().value(); + comments.addAll(((ConfigNodeField) node).comments()); + + // path must be on-stack while we parse the value + pathStack.push(path); + if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) { + // we really should make this work, but for now throwing + // an exception is better than producing an incorrect + // result. See + // https://github.com/lightbend/config/issues/160 + if (arrayCount > 0) { + throw parseError("Due to current limitations of the config parser, += does not work nested inside a list. " + + "+= expands to a ${} substitution and the path in ${} cannot currently refer to list elements. " + + "You might be able to move the += outside of the list and then refer to it from inside the list with ${}."); + } + + // because we will put it in an array after the fact so + // we want this to be incremented during the parseValue + // below in order to throw the above exception. + arrayCount += 1; + } + + AbstractConfigNodeValue valueNode; + AbstractConfigValue newValue; + + valueNode = ((ConfigNodeField) node).value(); + + // comments from the key token go to the value token + newValue = parseValue(valueNode, comments); + + if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) { + arrayCount -= 1; + + List concat = new ArrayList<>(2); + AbstractConfigValue previousRef = new ConfigReference(newValue.origin(), + new SubstitutionExpression(fullCurrentPath(), true /* optional */)); + AbstractConfigValue list = new SimpleConfigList(newValue.origin(), + Collections.singletonList(newValue)); + concat.add(previousRef); + concat.add(list); + newValue = ConfigConcatenation.concatenate(concat); + } + + // Grab any trailing comments on the same line + if (i < nodes.size() - 1) { + i++; + while (i < nodes.size()) { + if (nodes.get(i) instanceof ConfigNodeComment) { + ConfigNodeComment comment = (ConfigNodeComment) nodes.get(i); + newValue = newValue.withOrigin(newValue.origin().appendComments( + Collections.singletonList(comment.commentText()))); + break; + } else if (nodes.get(i) instanceof ConfigNodeSingleToken) { + ConfigNodeSingleToken curr = (ConfigNodeSingleToken) nodes.get(i); + if (curr.token() == Tokens.COMMA || Tokens.isIgnoredWhitespace(curr.token())) { + // keep searching, as there could still be a comment + } else { + i--; + break; + } + } else { + i--; + break; + } + i++; + } + } + + pathStack.pop(); + + String key = path.first(); + Path remaining = path.remainder(); + + if (remaining == null) { + AbstractConfigValue existing = values.get(key); + if (existing != null) { + // In strict JSON, dups should be an error; while in + // our custom config language, they should be merged + // if the value is an object (or substitution that + // could become an object). + + if (flavor == ConfigSyntax.JSON) { + throw parseError("JSON does not allow duplicate fields: '" + + key + + "' was already seen at " + + existing.origin().description()); + } else { + newValue = newValue.withFallback(existing); + } + } + values.put(key, newValue); + } else { + if (flavor == ConfigSyntax.JSON) { + throw new ConfigException.BugOrBroken( + "somehow got multi-element path in JSON mode"); + } + + AbstractConfigObject obj = createValueUnderPath( + remaining, newValue); + AbstractConfigValue existing = values.get(key); + if (existing != null) { + obj = obj.withFallback(existing); + } + values.put(key, obj); + } + } + } + + return new SimpleConfigObject(objectOrigin, values); + } + + private SimpleConfigList parseArray(ConfigNodeArray n) { + arrayCount += 1; + + SimpleConfigOrigin arrayOrigin = lineOrigin(); + List values = new ArrayList<>(); + + boolean lastWasNewLine = false; + List comments = new ArrayList<>(); + + AbstractConfigValue v = null; + + for (AbstractConfigNode node : n.children()) { + if (node instanceof ConfigNodeComment) { + comments.add(((ConfigNodeComment) node).commentText()); + lastWasNewLine = false; + } else if (node instanceof ConfigNodeSingleToken && Tokens.isNewline(((ConfigNodeSingleToken) node).token())) { + lineNumber++; + if (lastWasNewLine && v == null) { + comments.clear(); + } else if (v != null) { + values.add(v.withOrigin(v.origin().appendComments(new ArrayList<>(comments)))); + comments.clear(); + v = null; + } + lastWasNewLine = true; + } else if (node instanceof AbstractConfigNodeValue) { + lastWasNewLine = false; + if (v != null) { + values.add(v.withOrigin(v.origin().appendComments(new ArrayList<>(comments)))); + comments.clear(); + } + v = parseValue((AbstractConfigNodeValue) node, comments); + } + } + // There shouldn't be any comments at this point, but add them just in case + if (v != null) { + values.add(v.withOrigin(v.origin().appendComments(new ArrayList<>(comments)))); + } + arrayCount -= 1; + return new SimpleConfigList(arrayOrigin, values); + } + + AbstractConfigValue parse() { + AbstractConfigValue result = null; + ArrayList comments = new ArrayList<>(); + boolean lastWasNewLine = false; + for (AbstractConfigNode node : document.children()) { + if (node instanceof ConfigNodeComment) { + comments.add(((ConfigNodeComment) node).commentText()); + lastWasNewLine = false; + } else if (node instanceof ConfigNodeSingleToken) { + Token t = ((ConfigNodeSingleToken) node).token(); + if (Tokens.isNewline(t)) { + lineNumber++; + if (lastWasNewLine && result == null) { + comments.clear(); + } else if (result != null) { + result = result.withOrigin(result.origin().appendComments(new ArrayList<>(comments))); + comments.clear(); + break; + } + lastWasNewLine = true; + } + } else if (node instanceof ConfigNodeComplexValue) { + result = parseValue((ConfigNodeComplexValue) node, comments); + lastWasNewLine = false; + } + } + return result; + } + } +} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/Path.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/Path.java new file mode 100644 index 00000000000..8ec318f2aa4 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/Path.java @@ -0,0 +1,242 @@ +/* + * Copyright (C) 2011-2012 Typesafe Inc. + */ + +package org.apache.seatunnel.shade.com.typesafe.config.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions; + +import java.util.Iterator; +import java.util.List; + +final class Path { + + private final String first; + private final Path remainder; + private static final int DEFAULT_VALUE = 41; + + Path(String first, Path remainder) { + this.first = first; + this.remainder = remainder; + } + + Path(String... elements) { + if (elements.length == 0) { + throw new ConfigException.BugOrBroken("empty path"); + } + this.first = elements[0]; + if (elements.length > 1) { + PathBuilder pb = new PathBuilder(); + for (int i = 1; i < elements.length; ++i) { + pb.appendKey(elements[i]); + } + this.remainder = pb.result(); + } else { + this.remainder = null; + } + } + + // append all the paths in the list together into one path + Path(List pathsToConcat) { + this(pathsToConcat.iterator()); + } + + // append all the paths in the iterator together into one path + Path(Iterator i) { + if (!i.hasNext()) { + throw new ConfigException.BugOrBroken("empty path"); + } + + Path firstPath = i.next(); + this.first = firstPath.first; + + PathBuilder pb = new PathBuilder(); + if (firstPath.remainder != null) { + pb.appendPath(firstPath.remainder); + } + while (i.hasNext()) { + pb.appendPath(i.next()); + } + this.remainder = pb.result(); + } + + String first() { + return first; + } + + /** + * @return path minus the first element or null if no more elements + */ + Path remainder() { + return remainder; + } + + /** + * @return path minus the last element or null if we have just one element + */ + Path parent() { + if (remainder == null) { + return null; + } + + PathBuilder pb = new PathBuilder(); + Path p = this; + while (p.remainder != null) { + pb.appendKey(p.first); + p = p.remainder; + } + return pb.result(); + } + + /** + * @return last element in the path + */ + String last() { + Path p = this; + while (p.remainder != null) { + p = p.remainder; + } + return p.first; + } + + Path prepend(Path toPrepend) { + PathBuilder pb = new PathBuilder(); + pb.appendPath(toPrepend); + pb.appendPath(this); + return pb.result(); + } + + int length() { + int count = 1; + Path p = remainder; + while (p != null) { + count += 1; + p = p.remainder; + } + return count; + } + + Path subPath(int removeFromFront) { + int count = removeFromFront; + Path p = this; + while (p != null && count > 0) { + count -= 1; + p = p.remainder; + } + return p; + } + + Path subPath(int firstIndex, int lastIndex) { + if (lastIndex < firstIndex) { + throw new ConfigException.BugOrBroken("bad call to subPath"); + } + + Path from = subPath(firstIndex); + PathBuilder pb = new PathBuilder(); + int count = lastIndex - firstIndex; + while (count > 0) { + count -= 1; + pb.appendKey(from.first()); + from = from.remainder(); + if (from == null) { + throw new ConfigException.BugOrBroken("subPath lastIndex out of range " + lastIndex); + } + } + return pb.result(); + } + + boolean startsWith(Path other) { + Path myRemainder = this; + Path otherRemainder = other; + if (otherRemainder.length() <= myRemainder.length()) { + while (otherRemainder != null) { + if (!otherRemainder.first().equals(myRemainder.first())) { + return false; + } + myRemainder = myRemainder.remainder(); + otherRemainder = otherRemainder.remainder(); + } + return true; + } + return false; + } + + @Override + public boolean equals(Object other) { + if (other instanceof Path) { + Path that = (Path) other; + return this.first.equals(that.first) + && ConfigImplUtil.equalsHandlingNull(this.remainder, + that.remainder); + } else { + return false; + } + } + + @Override + public int hashCode() { + return DEFAULT_VALUE * (DEFAULT_VALUE + first.hashCode()) + + (remainder == null ? 0 : remainder.hashCode()); + } + + // this doesn't have a very precise meaning, just to reduce + // noise from quotes in the rendered path for average cases + static boolean hasFunkyChars(String s) { + int length = s.length(); + + if (length == 0) { + return false; + } + + for (int i = 0; i < length; ++i) { + char c = s.charAt(i); + + if (Character.isLetterOrDigit(c) || c == '-' || c == '_' || c == '.') { + continue; + } else { + return true; + } + } + return false; + } + + private void appendToStringBuilder(StringBuilder sb) { + if (hasFunkyChars(first) || first.isEmpty()) { + sb.append(ConfigImplUtil.renderJsonString(first)); + } else { + sb.append(first); + } + if (remainder != null) { + sb.append(ConfigParseOptions.PATH_TOKEN_SEPARATOR); + remainder.appendToStringBuilder(sb); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Path("); + appendToStringBuilder(sb); + sb.append(")"); + return sb.toString(); + } + + /** + * toString() is a debugging-oriented version while this is an + * error-message-oriented human-readable one. + */ + String render() { + StringBuilder sb = new StringBuilder(); + appendToStringBuilder(sb); + return sb.toString(); + } + + static Path newKey(String key) { + return new Path(key, null); + } + + static Path newPath(String path) { + return PathParser.parsePath(path); + } +} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PathParser.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PathParser.java new file mode 100644 index 00000000000..b512cd1cc44 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PathParser.java @@ -0,0 +1,296 @@ +/* + * Copyright (C) 2011-2012 Typesafe Inc. + */ + +package org.apache.seatunnel.shade.com.typesafe.config.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigOrigin; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigSyntax; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +final class PathParser { + + static ConfigOrigin API_ORIGIN = SimpleConfigOrigin.newSimple("path parameter"); + + static ConfigNodePath parsePathNode(String path) { + return parsePathNode(path, ConfigSyntax.CONF); + } + + static ConfigNodePath parsePathNode(String path, ConfigSyntax flavor) { + try (StringReader reader = new StringReader(path)) { + Iterator tokens = Tokenizer.tokenize(API_ORIGIN, reader, flavor); + tokens.next(); // drop START + return parsePathNodeExpression(tokens, API_ORIGIN, path, flavor); + } + } + + static Path parsePath(String path) { + Path speculated = speculativeFastParsePath(path); + if (speculated != null) { + return speculated; + } + try (StringReader reader = new StringReader(path)) { + Iterator tokens = Tokenizer.tokenize(API_ORIGIN, reader, ConfigSyntax.CONF); + tokens.next(); // drop START + return parsePathExpression(tokens, API_ORIGIN, path); + } + } + + protected static Path parsePathExpression(Iterator expression, + ConfigOrigin origin) { + return parsePathExpression(expression, origin, null, null, ConfigSyntax.CONF); + } + + protected static Path parsePathExpression(Iterator expression, + ConfigOrigin origin, String originalText) { + return parsePathExpression(expression, origin, originalText, null, ConfigSyntax.CONF); + } + + protected static ConfigNodePath parsePathNodeExpression(Iterator expression, + ConfigOrigin origin) { + return parsePathNodeExpression(expression, origin, null, ConfigSyntax.CONF); + } + + protected static ConfigNodePath parsePathNodeExpression(Iterator expression, + ConfigOrigin origin, String originalText, ConfigSyntax flavor) { + ArrayList pathTokens = new ArrayList<>(); + Path path = parsePathExpression(expression, origin, originalText, pathTokens, flavor); + return new ConfigNodePath(path, pathTokens); + } + + // originalText may be null if not available + protected static Path parsePathExpression(Iterator expression, + ConfigOrigin origin, String originalText, + ArrayList pathTokens, + ConfigSyntax flavor) { + // each builder in "buf" is an element in the path. + List buf = new ArrayList<>(); + buf.add(new Element("", false)); + + if (!expression.hasNext()) { + throw new ConfigException.BadPath(origin, originalText, + "Expecting a field name or path here, but got nothing"); + } + + while (expression.hasNext()) { + Token t = expression.next(); + + if (pathTokens != null) { + pathTokens.add(t); + } + + // Ignore all IgnoredWhitespace tokens + if (Tokens.isIgnoredWhitespace(t)) { + continue; + } + + if (Tokens.isValueWithType(t, ConfigValueType.STRING)) { + AbstractConfigValue v = Tokens.getValue(t); + // this is a quoted string; so any periods + // in here don't count as path separators + String s = v.transformToString(); + + addPathText(buf, true, s); + } else if (t == Tokens.END) { + // ignore this; when parsing a file, it should not happen + // since we're parsing a token list rather than the main + // token iterator, and when parsing a path expression from the + // API, it's expected to have an END. + } else { + // any periods outside of a quoted string count as + // separators + String text; + if (Tokens.isValue(t)) { + // appending a number here may add + // a period, but we _do_ count those as path + // separators, because we basically want + // "foo 3.0bar" to parse as a string even + // though there's a number in it. The fact that + // we tokenize non-string values is largely an + // implementation detail. + AbstractConfigValue v = Tokens.getValue(t); + + // We need to split the tokens on a . so that we can get sub-paths but still preserve + // the original path text when doing an insertion + if (pathTokens != null) { + pathTokens.remove(pathTokens.size() - 1); + pathTokens.addAll(splitTokenOnPeriod(t, flavor)); + } + text = v.transformToString(); + } else if (Tokens.isUnquotedText(t)) { + // We need to split the tokens on a . so that we can get sub-paths but still preserve + // the original path text when doing an insertion on ConfigNodeObjects + if (pathTokens != null) { + pathTokens.remove(pathTokens.size() - 1); + pathTokens.addAll(splitTokenOnPeriod(t, flavor)); + } + text = Tokens.getUnquotedText(t); + } else { + throw new ConfigException.BadPath( + origin, + originalText, + "Token not allowed in path expression: " + + t + + " (you can double-quote this token if you really want it here)"); + } + + addPathText(buf, false, text); + } + } + + PathBuilder pb = new PathBuilder(); + for (Element e : buf) { + if (e.sb.length() == 0 && !e.canBeEmpty) { + throw new ConfigException.BadPath( + origin, + originalText, + "path has a leading, trailing, or two adjacent period '.' (use quoted \"\" empty string if you want an empty element)"); + } else { + pb.appendKey(e.sb.toString()); + } + } + + return pb.result(); + } + + private static Collection splitTokenOnPeriod(Token t, ConfigSyntax flavor) { + + String tokenText = t.tokenText(); + if (tokenText.equals(ConfigParseOptions.PATH_TOKEN_SEPARATOR)) { + return Collections.singletonList(t); + } + String[] splitToken = tokenText.split(ConfigParseOptions.PATH_TOKEN_SEPARATOR); + ArrayList splitTokens = new ArrayList<>(); + for (String s : splitToken) { + if (flavor == ConfigSyntax.CONF) { + splitTokens.add(Tokens.newUnquotedText(t.origin(), s)); + } else { + splitTokens.add(Tokens.newString(t.origin(), s, "\"" + s + "\"")); + } + splitTokens.add(Tokens.newUnquotedText(t.origin(), ConfigParseOptions.PATH_TOKEN_SEPARATOR)); + } + + if (!tokenText.startsWith(ConfigParseOptions.PATH_TOKEN_SEPARATOR, tokenText.length() - ConfigParseOptions.PATH_TOKEN_SEPARATOR.length())) { + splitTokens.remove(splitTokens.size() - 1); + } + + return splitTokens; + } + + private static void addPathText(List buf, boolean wasQuoted, + String newText) { + + int i = wasQuoted ? -1 : newText.indexOf(ConfigParseOptions.PATH_TOKEN_SEPARATOR); + Element current = buf.get(buf.size() - 1); + if (i < 0) { + // add to current path element + current.sb.append(newText); + // any empty quoted string means this element can + // now be empty. + if (wasQuoted && current.sb.length() == 0) { + current.canBeEmpty = true; + } + } else { + // "buf" plus up to the period is an element + current.sb.append(newText, 0, i); + // then start a new element + buf.add(new Element("", false)); + // recurse to consume remainder of newText + addPathText(buf, false, newText.substring(i + ConfigParseOptions.PATH_TOKEN_SEPARATOR.length())); + } + } + + // the idea is to see if the string has any chars or features + // that might require the full parser to deal with. + private static boolean looksUnsafeForFastParser(String s) { + // TODO: maybe we should rewrite this function using ConfigParseOptions.pathTokenSeparator + boolean lastWasDot = true; // start of path is also a "dot" + int len = s.length(); + if (s.isEmpty()) { + return true; + } + if (s.charAt(0) == '.') { + return true; + } + if (s.charAt(len - 1) == '.') { + return true; + } + + for (int i = 0; i < len; ++i) { + char c = s.charAt(i); + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { + lastWasDot = false; + } else if (c == '.') { + if (lastWasDot) { + return true; // ".." means we need to throw an error + } + lastWasDot = true; + } else if (c == '-') { + if (lastWasDot) { + return true; + } + } else { + return true; + } + } + + if (lastWasDot) { + return true; + } + + return false; + } + + private static Path fastPathBuild(Path tail, String s, int end) { + + // lastIndexOf takes last index it should look at, end - 1 not end + int splitAt = s.lastIndexOf(ConfigParseOptions.PATH_TOKEN_SEPARATOR, end - 1); + ArrayList tokens = new ArrayList<>(); + tokens.add(Tokens.newUnquotedText(null, s)); + // this works even if splitAt is -1; then we start the substring at 0 + + if (splitAt < 0) { + Path withOneMoreElement = new Path(s.substring(0, end), tail); + return withOneMoreElement; + } else { + Path withOneMoreElement = new Path(s.substring(splitAt + ConfigParseOptions.PATH_TOKEN_SEPARATOR.length(), end), tail); + return fastPathBuild(withOneMoreElement, s, splitAt); + } + } + + // do something much faster than the full parser if + // we just have something like "foo" or "foo.bar" + private static Path speculativeFastParsePath(String path) { + String s = ConfigImplUtil.unicodeTrim(path); + if (looksUnsafeForFastParser(s)) { + return null; + } + + return fastPathBuild(null, s, s.length()); + } + + static class Element { + StringBuilder sb; + // an element can be empty if it has a quoted empty string "" in it + boolean canBeEmpty; + + Element(String initial, boolean canBeEmpty) { + this.canBeEmpty = canBeEmpty; + this.sb = new StringBuilder(initial); + } + + @Override + public String toString() { + return "Element(" + sb.toString() + "," + canBeEmpty + ")"; + } + } +} diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/SimpleConfigObject.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/SimpleConfigObject.java new file mode 100644 index 00000000000..ed52f4b4105 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/SimpleConfigObject.java @@ -0,0 +1,568 @@ +/* + * Copyright (C) 2011-2012 Typesafe Inc. + */ + +package org.apache.seatunnel.shade.com.typesafe.config.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.ConfigException; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigOrigin; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; + +import java.io.ObjectStreamException; +import java.io.Serializable; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +final class SimpleConfigObject extends AbstractConfigObject implements Serializable { + private static final long serialVersionUID = 2L; + private final Map value; + private final boolean resolved; + private final boolean ignoresFallbacks; + private static final SimpleConfigObject EMPTY_INSTANCE = empty(SimpleConfigOrigin.newSimple("empty config")); + private static final int HASH_CODE = 41; + + SimpleConfigObject(ConfigOrigin origin, Map value, ResolveStatus status, boolean ignoresFallbacks) { + super(origin); + if (value == null) { + throw new ConfigException.BugOrBroken("creating config object with null map"); + } else { + this.value = value; + this.resolved = status == ResolveStatus.RESOLVED; + this.ignoresFallbacks = ignoresFallbacks; + if (status != ResolveStatus.fromValues(value.values())) { + throw new ConfigException.BugOrBroken("Wrong resolved status on " + this); + } + } + } + + SimpleConfigObject(ConfigOrigin origin, Map value) { + this(origin, value, ResolveStatus.fromValues(value.values()), false); + } + + public SimpleConfigObject withOnlyKey(String key) { + return this.withOnlyPath(Path.newKey(key)); + } + + public SimpleConfigObject withoutKey(String key) { + return this.withoutPath(Path.newKey(key)); + } + + protected SimpleConfigObject withOnlyPathOrNull(Path path) { + String key = path.first(); + Path next = path.remainder(); + AbstractConfigValue v = this.value.get(key); + if (next != null) { + if (v instanceof AbstractConfigObject) { + v = ((AbstractConfigObject) v).withOnlyPathOrNull(next); + } else { + v = null; + } + } + + return v == null ? null : new SimpleConfigObject(this.origin(), Collections.singletonMap(key, v), v.resolveStatus(), this.ignoresFallbacks); + } + + SimpleConfigObject withOnlyPath(Path path) { + SimpleConfigObject o = this.withOnlyPathOrNull(path); + return o == null ? new SimpleConfigObject(this.origin(), Collections.emptyMap(), ResolveStatus.RESOLVED, this.ignoresFallbacks) : o; + } + + SimpleConfigObject withoutPath(Path path) { + String key = path.first(); + Path next = path.remainder(); + AbstractConfigValue v = this.value.get(key); + HashMap smaller; + if (next != null && v instanceof AbstractConfigObject) { + v = ((AbstractConfigObject) v).withoutPath(next); + smaller = new HashMap<>(this.value); + smaller.put(key, v); + return new SimpleConfigObject(this.origin(), smaller, ResolveStatus.fromValues(smaller.values()), this.ignoresFallbacks); + } else if (next == null && v != null) { + smaller = new HashMap<>(this.value.size() - 1); + + for (Entry stringAbstractConfigValueEntry : this.value.entrySet()) { + if (!stringAbstractConfigValueEntry.getKey().equals(key)) { + smaller.put(stringAbstractConfigValueEntry.getKey(), stringAbstractConfigValueEntry.getValue()); + } + } + + return new SimpleConfigObject(this.origin(), smaller, ResolveStatus.fromValues(smaller.values()), this.ignoresFallbacks); + } else { + return this; + } + } + + public SimpleConfigObject withValue(String key, ConfigValue v) { + if (v == null) { + throw new ConfigException.BugOrBroken("Trying to store null ConfigValue in a ConfigObject"); + } else { + Map newMap; + if (this.value.isEmpty()) { + newMap = Collections.singletonMap(key, (AbstractConfigValue) v); + } else { + newMap = new HashMap<>(this.value); + newMap.put(key, v); + } + + return new SimpleConfigObject(this.origin(), newMap, ResolveStatus.fromValues(newMap.values()), this.ignoresFallbacks); + } + } + + SimpleConfigObject withValue(Path path, ConfigValue v) { + String key = path.first(); + Path next = path.remainder(); + if (next == null) { + return this.withValue(key, v); + } else { + AbstractConfigValue child = this.value.get(key); + if (child instanceof AbstractConfigObject) { + return this.withValue(key, ((AbstractConfigObject) child).withValue(next, v)); + } else { + SimpleConfig subtree = ((AbstractConfigValue) v).atPath(SimpleConfigOrigin.newSimple("withValue(" + next.render() + ")"), next); + return this.withValue(key, subtree.root()); + } + } + } + + protected AbstractConfigValue attemptPeekWithPartialResolve(String key) { + return this.value.get(key); + } + + private SimpleConfigObject newCopy(ResolveStatus newStatus, ConfigOrigin newOrigin, boolean newIgnoresFallbacks) { + return new SimpleConfigObject(newOrigin, this.value, newStatus, newIgnoresFallbacks); + } + + protected SimpleConfigObject newCopy(ResolveStatus newStatus, ConfigOrigin newOrigin) { + return this.newCopy(newStatus, newOrigin, this.ignoresFallbacks); + } + + protected SimpleConfigObject withFallbacksIgnored() { + return this.ignoresFallbacks ? this : this.newCopy(this.resolveStatus(), this.origin(), true); + } + + ResolveStatus resolveStatus() { + return ResolveStatus.fromBoolean(this.resolved); + } + + public SimpleConfigObject replaceChild(AbstractConfigValue child, AbstractConfigValue replacement) { + Map newChildren = new HashMap<>(this.value); + Iterator> var4 = newChildren.entrySet().iterator(); + + Entry old; + do { + if (!var4.hasNext()) { + throw new ConfigException.BugOrBroken("SimpleConfigObject.replaceChild did not find " + child + " in " + this); + } + + old = var4.next(); + } while (old.getValue() != child); + + if (replacement != null) { + old.setValue(replacement); + } else { + newChildren.remove(old.getKey()); + } + + return new SimpleConfigObject(this.origin(), newChildren, ResolveStatus.fromValues(newChildren.values()), this.ignoresFallbacks); + } + + public boolean hasDescendant(AbstractConfigValue descendant) { + Iterator var2 = this.value.values().iterator(); + + AbstractConfigValue child; + do { + if (!var2.hasNext()) { + var2 = this.value.values().iterator(); + + do { + if (!var2.hasNext()) { + return false; + } + + child = var2.next(); + } while (!(child instanceof Container) || !((Container) child).hasDescendant(descendant)); + + return true; + } + + child = var2.next(); + } while (child != descendant); + + return true; + } + + protected boolean ignoresFallbacks() { + return this.ignoresFallbacks; + } + + public Map unwrapped() { + Map m = new HashMap<>(); + + for (Entry stringAbstractConfigValueEntry : this.value.entrySet()) { + m.put(stringAbstractConfigValueEntry.getKey(), stringAbstractConfigValueEntry.getValue().unwrapped()); + } + + return m; + } + + protected SimpleConfigObject mergedWithObject(AbstractConfigObject abstractFallback) { + this.requireNotIgnoringFallbacks(); + if (!(abstractFallback instanceof SimpleConfigObject)) { + throw new ConfigException.BugOrBroken("should not be reached (merging non-SimpleConfigObject)"); + } else { + SimpleConfigObject fallback = (SimpleConfigObject) abstractFallback; + boolean changed = false; + boolean allResolved = true; + Map merged = new HashMap<>(); + Set allKeys = new HashSet<>(); + allKeys.addAll(this.keySet()); + allKeys.addAll(fallback.keySet()); + + for (String key : allKeys) { + AbstractConfigValue first = this.value.get(key); + AbstractConfigValue second = fallback.value.get(key); + AbstractConfigValue kept; + if (first == null) { + kept = second; + } else if (second == null) { + kept = first; + } else { + kept = first.withFallback(second); + } + + merged.put(key, kept); + if (first != kept) { + changed = true; + } + + if (kept.resolveStatus() == ResolveStatus.UNRESOLVED) { + allResolved = false; + } + } + + ResolveStatus newResolveStatus = ResolveStatus.fromBoolean(allResolved); + boolean newIgnoresFallbacks = fallback.ignoresFallbacks(); + if (changed) { + return new SimpleConfigObject(mergeOrigins(this, fallback), merged, newResolveStatus, newIgnoresFallbacks); + } else if (newResolveStatus == this.resolveStatus() && newIgnoresFallbacks == this.ignoresFallbacks()) { + return this; + } else { + return this.newCopy(newResolveStatus, this.origin(), newIgnoresFallbacks); + } + } + } + + private SimpleConfigObject modify(NoExceptionsModifier modifier) { + try { + return this.modifyMayThrow(modifier); + } catch (RuntimeException var3) { + throw var3; + } catch (Exception var4) { + throw new ConfigException.BugOrBroken("unexpected checked exception", var4); + } + } + + private SimpleConfigObject modifyMayThrow(Modifier modifier) throws Exception { + Map changes = null; + + for (String k : this.keySet()) { + AbstractConfigValue v = this.value.get(k); + AbstractConfigValue modified = modifier.modifyChildMayThrow(k, v); + if (modified != v) { + if (changes == null) { + changes = new HashMap<>(); + } + + changes.put(k, modified); + } + } + + if (changes == null) { + return this; + } else { + Map modified = new HashMap<>(); + boolean sawUnresolved = false; + + for (String k : this.keySet()) { + AbstractConfigValue newValue; + if (changes.containsKey(k)) { + newValue = changes.get(k); + if (newValue != null) { + modified.put(k, newValue); + if (newValue.resolveStatus() == ResolveStatus.UNRESOLVED) { + sawUnresolved = true; + } + } + } else { + newValue = this.value.get(k); + modified.put(k, newValue); + if (newValue.resolveStatus() == ResolveStatus.UNRESOLVED) { + sawUnresolved = true; + } + } + } + + return new SimpleConfigObject(this.origin(), modified, sawUnresolved ? ResolveStatus.UNRESOLVED : ResolveStatus.RESOLVED, this.ignoresFallbacks()); + } + } + + ResolveResult resolveSubstitutions(ResolveContext context, ResolveSource source) throws NotPossibleToResolve { + if (this.resolveStatus() == ResolveStatus.RESOLVED) { + return ResolveResult.make(context, this); + } else { + ResolveSource sourceWithParent = source.pushParent(this); + + try { + SimpleConfigObject.ResolveModifier modifier = new SimpleConfigObject.ResolveModifier(context, sourceWithParent); + AbstractConfigValue value = this.modifyMayThrow(modifier); + return ResolveResult.make(modifier.context, value).asObjectResult(); + } catch (NotPossibleToResolve | RuntimeException var6) { + throw var6; + } catch (Exception var8) { + throw new ConfigException.BugOrBroken("unexpected checked exception", var8); + } + } + } + + SimpleConfigObject relativized(final Path prefix) { + return this.modify(new NoExceptionsModifier() { + public AbstractConfigValue modifyChild(String key, AbstractConfigValue v) { + return v.relativized(prefix); + } + }); + } + + protected void render(StringBuilder sb, int indent, boolean atRoot, ConfigRenderOptions options) { + if (this.isEmpty()) { + sb.append("{}"); + } else { + boolean outerBraces = options.getJson() || !atRoot; + int innerIndent; + if (outerBraces) { + innerIndent = indent + 1; + sb.append("{"); + if (options.getFormatted()) { + sb.append('\n'); + } + } else { + innerIndent = indent; + } + + int separatorCount = 0; + String[] keys = this.keySet().toArray(new String[0]); + + for (String k : keys) { + AbstractConfigValue v = this.value.get(k); + if (options.getOriginComments()) { + String[] lines = v.origin().description().split("\n"); + + for (String l : lines) { + indent(sb, indent + 1, options); + sb.append('#'); + if (!l.isEmpty()) { + sb.append(' '); + } + + sb.append(l); + sb.append("\n"); + } + } + + if (options.getComments()) { + + for (String comment : v.origin().comments()) { + indent(sb, innerIndent, options); + sb.append("#"); + if (!comment.startsWith(" ")) { + sb.append(' '); + } + + sb.append(comment); + sb.append("\n"); + } + } + + indent(sb, innerIndent, options); + v.render(sb, innerIndent, false, k, options); + if (options.getFormatted()) { + if (options.getJson()) { + sb.append(","); + separatorCount = 2; + } else { + separatorCount = 1; + } + + sb.append('\n'); + } else { + sb.append(","); + separatorCount = 1; + } + } + + sb.setLength(sb.length() - separatorCount); + if (outerBraces) { + if (options.getFormatted()) { + sb.append('\n'); + indent(sb, indent, options); + } + + sb.append("}"); + } + } + + if (atRoot && options.getFormatted()) { + sb.append('\n'); + } + + } + + public AbstractConfigValue get(Object key) { + return this.value.get(key); + } + + private static boolean mapEquals(Map a, Map b) { + if (a == b) { + return true; + } else { + Set aKeys = a.keySet(); + Set bKeys = b.keySet(); + if (aKeys.equals(bKeys)) { + Iterator var4 = aKeys.iterator(); + + String key; + do { + if (!var4.hasNext()) { + return true; + } + + key = var4.next(); + } while (a.get(key).equals(b.get(key))); + + } + return false; + } + } + + @SuppressWarnings("magicnumber") + private static int mapHash(Map m) { + List keys = new ArrayList<>(m.keySet()); + Collections.sort(keys); + int valuesHash = 0; + + String k; + for (Iterator var3 = keys.iterator(); var3.hasNext(); valuesHash += m.get(k).hashCode()) { + k = var3.next(); + } + + return HASH_CODE * (HASH_CODE + keys.hashCode()) + valuesHash; + } + + protected boolean canEqual(Object other) { + return other instanceof ConfigObject; + } + + public boolean equals(Object other) { + if (!(other instanceof ConfigObject)) { + return false; + } else { + return this.canEqual(other) && mapEquals(this, (ConfigObject) other); + } + } + + public int hashCode() { + return mapHash(this); + } + + public boolean containsKey(Object key) { + return this.value.containsKey(key); + } + + public Set keySet() { + return this.value.keySet(); + } + + public boolean containsValue(Object v) { + return this.value.containsValue(v); + } + + public Set> entrySet() { + HashSet> entries = new HashSet<>(); + + for (Entry stringAbstractConfigValueEntry : this.value.entrySet()) { + entries.add(new AbstractMap.SimpleImmutableEntry<>(stringAbstractConfigValueEntry.getKey(), stringAbstractConfigValueEntry.getValue())); + } + + return entries; + } + + public boolean isEmpty() { + return this.value.isEmpty(); + } + + public int size() { + return this.value.size(); + } + + public Collection values() { + return new HashSet<>(this.value.values()); + } + + static SimpleConfigObject empty() { + return EMPTY_INSTANCE; + } + + static SimpleConfigObject empty(ConfigOrigin origin) { + return origin == null ? empty() : new SimpleConfigObject(origin, Collections.emptyMap()); + } + + static SimpleConfigObject emptyMissing(ConfigOrigin baseOrigin) { + return new SimpleConfigObject(SimpleConfigOrigin.newSimple(baseOrigin.description() + " (not found)"), Collections.emptyMap()); + } + + private Object writeReplace() throws ObjectStreamException { + return new SerializedConfigValue(this); + } + + private static final class ResolveModifier implements Modifier { + final Path originalRestrict; + ResolveContext context; + final ResolveSource source; + + ResolveModifier(ResolveContext context, ResolveSource source) { + this.context = context; + this.source = source; + this.originalRestrict = context.restrictToChild(); + } + + public AbstractConfigValue modifyChildMayThrow(String key, AbstractConfigValue v) throws NotPossibleToResolve { + if (this.context.isRestrictedToChild()) { + if (key.equals(this.context.restrictToChild().first())) { + Path remainder = this.context.restrictToChild().remainder(); + if (remainder != null) { + ResolveResult result = this.context.restrict(remainder).resolve(v, this.source); + this.context = result.context.unrestricted().restrict(this.originalRestrict); + return result.value; + } else { + return v; + } + } else { + return v; + } + } else { + ResolveResult result = this.context.unrestricted().resolve(v, this.source); + this.context = result.context.unrestricted().restrict(this.originalRestrict); + return result.value; + } + } + } +} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/CompleteTest.java b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/CompleteTest.java new file mode 100644 index 00000000000..1652b5d45c1 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/CompleteTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.config; + +import org.apache.seatunnel.config.utils.FileUtils; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; + +public class CompleteTest { + + @Test + public void testVariables() throws URISyntaxException { + // We use a map to mock the system property, since the system property will be only loaded once + // after the test is run. see Issue #1670 + Map systemProperties = new HashMap<>(); + systemProperties.put("dt", "20190318"); + systemProperties.put("city2", "shanghai"); + + Config config = ConfigFactory + .parseFile(FileUtils.getFileFromResources("/seatunnel/variables.conf")) + .resolveWith(ConfigFactory.parseMap(systemProperties), ConfigResolveOptions.defaults().setAllowUnresolved(true)); + String sql1 = config.getConfigList("transform").get(1).getString("sql"); + String sql2 = config.getConfigList("transform").get(2).getString("sql"); + + Assertions.assertTrue(sql1.contains("shanghai")); + Assertions.assertTrue(sql2.contains("20190318")); + + } + +} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java new file mode 100644 index 00000000000..4f24ab7afde --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/ConfigFactoryTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.config; + +import org.apache.seatunnel.config.utils.FileUtils; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.List; + +public class ConfigFactoryTest { + + @Test + public void testBasicParseAppConf() throws URISyntaxException { + + Config config = ConfigFactory.parseFile(FileUtils.getFileFromResources("/factory/config.conf")); + + Assertions.assertTrue(config.hasPath("env")); + Assertions.assertTrue(config.hasPath("source")); + Assertions.assertTrue(config.hasPath("transform")); + Assertions.assertTrue(config.hasPath("sink")); + + // check evn config + Config env = config.getConfig("env"); + Assertions.assertEquals("SeaTunnel", env.getString("spark.app.name")); + Assertions.assertEquals("2", env.getString("spark.executor.instances")); + Assertions.assertEquals("1", env.getString("spark.executor.cores")); + Assertions.assertEquals("1g", env.getString("spark.executor.memory")); + Assertions.assertEquals("5", env.getString("spark.stream.batchDuration")); + + // check custom plugin + Assertions.assertEquals("c.Console", config.getConfigList("sink").get(1).getString("plugin_name")); + } + + @Test + public void testTransformOrder() throws URISyntaxException { + + Config config = ConfigFactory.parseFile(FileUtils.getFileFromResources("/factory/config.conf")); + + String[] pluginNames = {"split", "sql1", "sql2", "sql3", "json"}; + + List transforms = config.getConfigList("transform"); + Assertions.assertEquals(pluginNames.length, transforms.size()); + + for (int i = 0; i < transforms.size(); i++) { + String parsedPluginName = String.valueOf(transforms.get(i).root().get("plugin_name").unwrapped()); + Assertions.assertEquals(pluginNames[i], parsedPluginName); + } + + } + + @Test + public void testQuotedString() throws URISyntaxException { + List keys = Arrays.asList("spark.app.name", "spark.executor.instances", "spark.executor.cores", + "spark.executor.memory", "spark.stream.batchDuration"); + + Config config = ConfigFactory.parseFile(FileUtils.getFileFromResources("/factory/config.conf")); + Config evnConfig = config.getConfig("env"); + evnConfig.entrySet().forEach(entry -> Assertions.assertTrue(keys.contains(entry.getKey()))); + + } +} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/JsonFormatTest.java b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/JsonFormatTest.java new file mode 100644 index 00000000000..bc39f5cd416 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/JsonFormatTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.config; + +import org.apache.seatunnel.config.utils.FileUtils; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.net.URISyntaxException; + +public class JsonFormatTest { + + @Test + public void testJsonFormat() throws URISyntaxException { + + Config json = ConfigFactory + .parseFile(FileUtils.getFileFromResources("/json/spark.batch.json")) + .resolveWith(ConfigFactory.systemProperties(), + ConfigResolveOptions.defaults().setAllowUnresolved(true)); + + Config config = ConfigFactory + .parseFile(FileUtils.getFileFromResources("/json/spark.batch.conf")) + .resolveWith(ConfigFactory.systemProperties(), + ConfigResolveOptions.defaults().setAllowUnresolved(true)); + + Assertions.assertEquals(config.atPath("transform"), json.atPath("transform")); + Assertions.assertEquals(config.atPath("sink"), json.atPath("sink")); + Assertions.assertEquals(config.atPath("source"), json.atPath("source")); + Assertions.assertEquals(config.atPath("env"), json.atPath("env")); + + } + +} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/utils/FileUtils.java b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/utils/FileUtils.java new file mode 100644 index 00000000000..3b0e01b79f9 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/test/java/org/apache/seatunnel/config/utils/FileUtils.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.config.utils; + +import java.io.File; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Paths; + +public final class FileUtils { + + private FileUtils() { + } + + // get file from classpath, resources folder + public static File getFileFromResources(String fileName) throws URISyntaxException { + URL resource = FileUtils.class.getResource(fileName); + if (resource == null) { + throw new IllegalArgumentException("file is not found!"); + } + return Paths.get(resource.toURI()).toFile(); + } + +} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf b/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf new file mode 100644 index 00000000000..af8c82b8ffa --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/test/resources/factory/config.conf @@ -0,0 +1,66 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +###### +###### This config file is a demonstration of batch processing in seatunnel config +###### + +env { + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + "spark.executor.cores" = 1 + "spark.executor.memory" = "1g" + "spark.stream.batchDuration" = 5 +} + +source { + + fakeStream { + content = ["Hello World, SeaTunnel"] + } + +} + +transform { + + split { + fields = ["msg", "name"] + delimiter = "," + } + + sql1 { + sql = "sql1" + } + + sql2 { + sql = "sql2" + } + + sql3 { + sql = "sql3" + } + + json { + sql = "sql3" + } + +} + +sink { + Console {} + c.Console {} +} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.conf b/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.conf new file mode 100644 index 00000000000..27ad42b4139 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +###### +###### This config file is a demonstration of batch processing in SeaTunnel config +###### + +env { + # You can set spark configuration here + # see available properties defined by spark: https://spark.apache.org/docs/latest/configuration.html#available-properties + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" +} + +source { + # This is a example input plugin **only for test and demonstrate the feature input plugin** + Fake { + result_table_name = "my_dataset" + } + + # You can also use other input plugins, such as hdfs + # hdfs { + # result_table_name = "accesslog" + # path = "hdfs://hadoop-cluster-01/nginx/accesslog" + # format = "json" + # } + + # If you would like to get more information about how to configure seatunnel and see full list of input plugins, + # please go to https://seatunnel.apache.org/docs/spark/configuration/source-plugins/Fake +} + +transform { + # split data by specific delimiter + + # you can also use other transform plugins, such as sql + # sql { + # sql = "select * from accesslog where request_time > 1000" + # } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/spark/configuration/transform-plugins/Split +} + +sink { + # choose stdout output plugin to output data to console + Console {} + + # you can also you other output plugins, such as sql + # hdfs { + # path = "hdfs://hadoop-cluster-01/nginx/accesslog_processed" + # save_mode = "append" + # } + + # If you would like to get more information about how to configure seatunnel and see full list of output plugins, + # please go to https://seatunnel.apache.org/docs/spark/configuration/sink-plugins/Console +} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.json b/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.json new file mode 100644 index 00000000000..f0f68aee576 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/test/resources/json/spark.batch.json @@ -0,0 +1,20 @@ +{ + "env" : { + "spark.app.name" : "SeaTunnel", + "spark.executor.cores" : 1, + "spark.executor.instances" : 2, + "spark.executor.memory" : "1g" + }, + "sink" : [ + { + "plugin_name" : "Console" + } + ], + "source" : [ + { + "plugin_name" : "Fake", + "result_table_name" : "my_dataset" + } + ], + "transform" : [] +} diff --git a/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf b/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf new file mode 100644 index 00000000000..f06fb1c1619 --- /dev/null +++ b/seatunnel-config/seatunnel-config-shade/src/test/resources/seatunnel/variables.conf @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +spark { + spark.stream.batchDuration = 5 + + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" +} + +source { + fakestream { + content = [ + "20190318, beijing, first message", + "20190319, shanghai, second message", + "20190318, shanghai, third message" + ] + rate = 1 + } +} + +transform { + split { + fields = ["dt", "city", "msg"] + delimiter = "," + } + + sql { + table_name = "user_view" + sql = "select * from user_view where city = '"${city2}"'" + result_table_name = "result1" + } + + sql { + table_name = "user_view" + sql = "select * from user_view where dt = '"${dt}"'" + result_table_name = "result2" + } +} + +sink { + stdout { + source_table_name="result1" + } + + stdout { + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index 59f9a7752f7..1e731e3d7cf 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -104,7 +104,7 @@ private void initializeJdbcConnection() throws SQLException, ClassNotFoundExcept * init the table */ private void initializeJdbcTable() { - URL resource = JdbcDb2IT.class.getResource("/jdbc/init/db2_init.conf"); + URL resource = JdbcDb2IT.class.getResource("/init/db2_init.conf"); if (resource == null) { throw new IllegalArgumentException("can't find find file"); } From 17c98deb8deccbfbde088515968605e7955c5f6b Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Mon, 17 Oct 2022 00:22:29 +0800 Subject: [PATCH 09/23] fix --- .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 2 +- .../resources}/jdbc_db2_source_and_sink.conf | 12 +- .../resources/jdbc/init_sql/db2_init.conf | 108 ------------------ .../jdbc/jdbc_db2_source_and_sink.conf | 89 --------------- 4 files changed, 3 insertions(+), 208 deletions(-) rename seatunnel-e2e/{seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc => seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources}/jdbc_db2_source_and_sink.conf (88%) delete mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/init_sql/db2_init.conf delete mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index 1e731e3d7cf..a907ce2c4d9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -146,7 +146,7 @@ void pullImageOK() throws SQLException { @DisplayName("JDBC-DM end to end test") public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException, SQLException { assertHasData(SOURCE_TABLE); - Container.ExecResult execResult = container.executeJob("/jdbc/jdbc_db2_source_and_sink.conf"); + Container.ExecResult execResult = container.executeJob("/jdbc_db2_source_and_sink.conf"); Assertions.assertEquals(0, execResult.getExitCode()); assertHasData(SINK_TABLE); } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf similarity index 88% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf index 5ca4a623415..829467bb4b0 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf @@ -6,7 +6,7 @@ # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -14,17 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### env { - # You can set spark configuration here - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + execution.parallelism = 1 job.mode = "BATCH" } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/init_sql/db2_init.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/init_sql/db2_init.conf deleted file mode 100644 index ee6ac9a0c1e..00000000000 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/init_sql/db2_init.conf +++ /dev/null @@ -1,108 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# db2 sql can't start with \n -table_source = """create table "DB2INST1".E2E_TABLE_SOURCE -( - COL_BOOLEAN BOOLEAN, - COL_INT INT, - COL_INTEGER INTEGER, - COL_SMALLINT SMALLINT, - COL_BIGINT BIGINT, - - COL_DECIMAL DECIMAL, - COL_DEC DEC, - COL_NUMERIC NUMERIC, - COL_NUMBER NUM, - - COL_TIMESTAMP TIMESTAMP, - COL_TIME TIME, - COL_DATE DATE, - - COL_REAL REAL, - COL_FLOAT FLOAT, - COL_DOUBLE_PRECISION DOUBLE PRECISION, - COL_DOUBLE DOUBLE, - COL_DECFLOAT DECFLOAT, - - COL_CHAR CHAR, - COL_VARCHAR VARCHAR(255), - COL_LONG_VARCHAR "LONG VARCHAR", - - - COL_GRAPHIC GRAPHIC(10), - COL_VARGRAPHIC VARGRAPHIC(1024), - COL_LONG_VARGRAPHIC "LONG VARGRAPHIC", - COL_DBCLOB DBCLOB, - - COL_BLOB BLOB, - COL_BINARY BINARY, - COL_VARBINARY VARBINARY(255), - COL_XML XML -) -""" - -table_sink = """create table "DB2INST1".E2E_TABLE_SINK -( - COL_BOOLEAN BOOLEAN, - COL_INT INT, - COL_INTEGER INTEGER, - COL_SMALLINT SMALLINT, - COL_BIGINT BIGINT, - - COL_DECIMAL DECIMAL, - COL_DEC DEC, - COL_NUMERIC NUMERIC, - COL_NUMBER NUM, - - COL_TIMESTAMP TIMESTAMP, - COL_TIME TIME, - COL_DATE DATE, - - COL_REAL REAL, - COL_FLOAT FLOAT, - COL_DOUBLE_PRECISION DOUBLE PRECISION, - COL_DOUBLE DOUBLE, - COL_DECFLOAT DECFLOAT, - - COL_CHAR CHAR, - COL_VARCHAR VARCHAR(255), - COL_LONG_VARCHAR "LONG VARCHAR", - - - COL_GRAPHIC GRAPHIC(10), - COL_VARGRAPHIC VARGRAPHIC(1024), - COL_LONG_VARGRAPHIC "LONG VARGRAPHIC", - COL_DBCLOB DBCLOB, - - COL_BLOB BLOB, - COL_BINARY BINARY, - COL_VARBINARY VARBINARY(255), - COL_XML XML -) -""" - -DML = """insert into "DB2INST1".E2E_TABLE_SOURCE - (COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, - COL_NUMERIC, COL_NUMBER, COL_TIMESTAMP, COL_TIME, COL_DATE, COL_REAL, COL_FLOAT, - COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, - COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC, - COL_DBCLOB, COL_BLOB, COL_BINARY, COL_VARBINARY, COL_XML) -VALUES (true, 1, 2, 3, 4, 5.0, 6.0, 7.0, 8.0, '2022-09-11 14:58:06.000000', null, '2022-09-11', 1.1, 1.1, 1.1, 1.1, 1.1, - 'a', 'varchar', 'longvarchar', 'GRAPHIC', 'var graphic', 'log graphic', null, null, null, null, - null) -""" diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf deleted file mode 100644 index 4060ea50b46..00000000000 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/src/test/resources/jdbc/jdbc_db2_source_and_sink.conf +++ /dev/null @@ -1,89 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### - -env { - # You can set flink configuration here - execution.parallelism = 1 - job.mode = "BATCH" - #execution.checkpoint.interval = 10000 - #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" -} - -source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** - Jdbc { - driver = com.ibm.db2.jcc.DB2Driver - url = "jdbc:db2://spark_e2e_db2:50000/testdb" - user = DB2INST1 - password = 123456 - query = """select COL_BOOLEAN, - COL_INT, - COL_INTEGER, - COL_SMALLINT, - COL_BIGINT, - COL_DECIMAL, - COL_DEC, - COL_NUMERIC, - COL_NUMBER, - COL_TIMESTAMP, - COL_DATE, - COL_REAL, - COL_FLOAT, - COL_DOUBLE_PRECISION, - COL_DOUBLE, - COL_DECFLOAT, - COL_CHAR, - COL_VARCHAR, - COL_LONG_VARCHAR, - COL_CLOB, - COL_GRAPHIC, - COL_VARGRAPHIC, - COL_LONG_VARGRAPHIC - from "DB2INST1".E2E_TABLE_SOURCE; - """ - } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc -} - -transform { - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql -} - -sink { - Jdbc { - driver = com.ibm.db2.jcc.DB2Driver - url = "jdbc:db2://spark_e2e_db2:50000/testdb" - user = DB2INST1 - password = 123456 - query = """insert into DB2INST1.E2E_TABLE_SOURCE(COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, - COL_NUMERIC, COL_NUMBER, COL_TIMESTAMP, COL_DATE, COL_REAL, COL_FLOAT, - COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, - COL_LONG_VARCHAR, COL_CLOB, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -""" - } - - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc -} From da770e0626fc64a86e1aad9680a18d2ac6314de0 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Mon, 17 Oct 2022 00:26:47 +0800 Subject: [PATCH 10/23] fix --- .../seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index a907ce2c4d9..fe99d9a82d2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -73,8 +73,7 @@ public void startUp() throws Exception { given().ignoreExceptions() .await() .atMost(180, TimeUnit.SECONDS) - .untilAsserted(this::initializeJdbcConnection); - initializeJdbcTable(); + .untilAsserted(this::initializeDbServer); } @Override @@ -87,7 +86,7 @@ public void tearDown() throws Exception { } } - private void initializeJdbcConnection() throws SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { + private void initializeDbServer() throws SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { Properties properties = new Properties(); properties.setProperty("user", USER); properties.setProperty("password", PASSWORD); @@ -98,6 +97,7 @@ private void initializeJdbcConnection() throws SQLException, ClassNotFoundExcept Assertions.assertTrue(resultSet.next()); resultSet.close(); statement.close(); + initializeJdbcTable(); } /** From 3f8d4d000408fa7300acd280e629a92c63b489d0 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Mon, 17 Oct 2022 00:47:54 +0800 Subject: [PATCH 11/23] fix --- .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index fe99d9a82d2..7b302ae85f9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -73,7 +73,8 @@ public void startUp() throws Exception { given().ignoreExceptions() .await() .atMost(180, TimeUnit.SECONDS) - .untilAsserted(this::initializeDbServer); + .untilAsserted(this::initializeJdbcConnection); + initializeJdbcTable(); } @Override @@ -86,7 +87,7 @@ public void tearDown() throws Exception { } } - private void initializeDbServer() throws SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { + private void initializeJdbcConnection() throws SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { Properties properties = new Properties(); properties.setProperty("user", USER); properties.setProperty("password", PASSWORD); @@ -97,7 +98,6 @@ private void initializeDbServer() throws SQLException, ClassNotFoundException, I Assertions.assertTrue(resultSet.next()); resultSet.close(); statement.close(); - initializeJdbcTable(); } /** @@ -127,7 +127,11 @@ private void initializeJdbcTable() { } } - private void assertHasData(String table) throws SQLException { + private void assertHasData(String table) throws SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { + if(jdbcConnection.isValid(10)){ + initializeJdbcConnection(); + } + try (Statement statement = jdbcConnection.createStatement()) { String sql = String.format("select * from \"%s\".%s", USER, table); ResultSet source = statement.executeQuery(sql); @@ -138,13 +142,13 @@ private void assertHasData(String table) throws SQLException { } @Test - void pullImageOK() throws SQLException { + void pullImageOK() throws SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { assertHasData(SOURCE_TABLE); } @TestTemplate @DisplayName("JDBC-DM end to end test") - public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException, SQLException { + public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException, SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { assertHasData(SOURCE_TABLE); Container.ExecResult execResult = container.executeJob("/jdbc_db2_source_and_sink.conf"); Assertions.assertEquals(0, execResult.getExitCode()); From 8fdbb8bfa2b75fcf9fcb524abf8f3c8d9fc1c044 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Mon, 17 Oct 2022 00:51:12 +0800 Subject: [PATCH 12/23] fix --- .../apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index 7b302ae85f9..c43a92312ca 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -147,7 +147,7 @@ void pullImageOK() throws SQLException, ClassNotFoundException, InstantiationExc } @TestTemplate - @DisplayName("JDBC-DM end to end test") + @DisplayName("JDBC-Db2 end to end test") public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException, SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { assertHasData(SOURCE_TABLE); Container.ExecResult execResult = container.executeJob("/jdbc_db2_source_and_sink.conf"); From 0f50f0256c3185e0fc690fe95e6ab426e6b1a9ed Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Tue, 18 Oct 2022 23:55:51 +0800 Subject: [PATCH 13/23] fix --- .../connector-jdbc/pom.xml | 7 +++- .../connector-jdbc-e2e/pom.xml | 5 +++ .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 33 +++++++++---------- .../resources/jdbc_db2_source_and_sink.conf | 12 +++---- .../connector-jdbc-spark-e2e/pom.xml | 6 ---- 5 files changed, 32 insertions(+), 31 deletions(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/pom.xml b/seatunnel-connectors-v2/connector-jdbc/pom.xml index bd22d6758ed..ed706fa5819 100644 --- a/seatunnel-connectors-v2/connector-jdbc/pom.xml +++ b/seatunnel-connectors-v2/connector-jdbc/pom.xml @@ -77,6 +77,12 @@ ${oracle.version} provided + + com.ibm.db2.jcc + db2jcc + ${db2.version} + provided + @@ -112,7 +118,6 @@ com.ibm.db2.jcc db2jcc - ${db2.version} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml index 2a65167729d..4ecc1975ce0 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml @@ -76,6 +76,11 @@ ojdbc8 test + + com.ibm.db2.jcc + db2jcc + test + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index c43a92312ca..b4692b895a7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -40,35 +40,36 @@ public class JdbcDb2IT extends TestSuiteBase implements TestResource { * db2 in dockerhub */ private static final String IMAGE = "ibmcom/db2:latest"; - private static final String HOST = "spark_e2e_db2"; + private static final String HOST = "e2e_db2"; private static final int PORT = 50000; - private static final int LOCAL_PORT = 50000; + private static final int LOCAL_PORT = 50001; private static final String USER = "DB2INST1"; private static final String PASSWORD = "123456"; private static final String DRIVER = "com.ibm.db2.jcc.DB2Driver"; - private static final String DATABASE = "testdb"; + private static final String DATABASE = "E2E"; private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE"; private static final String SINK_TABLE = "E2E_TABLE_SINK"; private String jdbcUrl; - private GenericContainer dbserver; + private GenericContainer dbServer; private Connection jdbcConnection; @BeforeAll @Override public void startUp() throws Exception { - dbserver = new GenericContainer<>(IMAGE) - .withNetwork(TestContainer.NETWORK) + dbServer = new GenericContainer<>(IMAGE) + .withNetwork(NETWORK) .withNetworkAliases(HOST) .withPrivilegedMode(true) .withLogConsumer(new Slf4jLogConsumer(LOG)) .withEnv("DB2INST1_PASSWORD", PASSWORD) .withEnv("DBNAME", DATABASE) .withEnv("LICENSE", "accept") + .withSharedMemorySize(4 * 1024 * 1024 * 1024L) ; - dbserver.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); - Startables.deepStart(Stream.of(dbserver)).join(); - jdbcUrl = String.format("jdbc:db2://%s:%s/%s", dbserver.getHost(), LOCAL_PORT, DATABASE); + dbServer.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); + Startables.deepStart(Stream.of(dbServer)).join(); + jdbcUrl = String.format("jdbc:db2://%s:%s/%s", dbServer.getHost(), LOCAL_PORT, DATABASE); LOG.info("DB2 container started"); given().ignoreExceptions() .await() @@ -82,8 +83,8 @@ public void tearDown() throws Exception { if (jdbcConnection != null) { jdbcConnection.close(); } - if (dbserver != null) { - dbserver.close(); + if (dbServer != null) { + dbServer.close(); } } @@ -127,11 +128,7 @@ private void initializeJdbcTable() { } } - private void assertHasData(String table) throws SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { - if(jdbcConnection.isValid(10)){ - initializeJdbcConnection(); - } - + private void assertHasData(String table) { try (Statement statement = jdbcConnection.createStatement()) { String sql = String.format("select * from \"%s\".%s", USER, table); ResultSet source = statement.executeQuery(sql); @@ -142,13 +139,13 @@ private void assertHasData(String table) throws SQLException, ClassNotFoundExcep } @Test - void pullImageOK() throws SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { + void pullImageOK() { assertHasData(SOURCE_TABLE); } @TestTemplate @DisplayName("JDBC-Db2 end to end test") - public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException, SQLException, ClassNotFoundException, InstantiationException, IllegalAccessException { + public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException { assertHasData(SOURCE_TABLE); Container.ExecResult execResult = container.executeJob("/jdbc_db2_source_and_sink.conf"); Assertions.assertEquals(0, execResult.getExitCode()); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf index 829467bb4b0..6a67a63fa84 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf @@ -24,9 +24,9 @@ source { # This is a example source plugin **only for test and demonstrate the feature source plugin** Jdbc { driver = com.ibm.db2.jcc.DB2Driver - url = "jdbc:db2://spark_e2e_db2:50000/testdb" - user = DB2INST1 - password = 123456 + url = "jdbc:db2://e2e_db2:50000/E2E" + user = "DB2INST1" + password = "123456" query = """ select COL_BOOLEAN, COL_INT, @@ -65,9 +65,9 @@ transform { sink { Jdbc { driver = com.ibm.db2.jcc.DB2Driver - url = "jdbc:db2://spark_e2e_db2:50000/testdb" - user = DB2INST1 - password = 123456 + url = "jdbc:db2://e2e_db2:50000/E2E" + user = "DB2INST1" + password = "123456" query = """ insert into "DB2INST1".E2E_TABLE_SINK(COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, COL_NUMERIC, COL_NUMBER, COL_REAL, COL_FLOAT, diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml index cd86cbb81f7..a0cb7eeadc3 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml @@ -96,12 +96,6 @@ com.microsoft.sqlserver mssql-jdbc - - com.ibm.db2.jcc - db2jcc - db2jcc4 - test - org.testcontainers From 3c8d1d9b7349a77d982d7634c2898944781776b3 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Fri, 21 Oct 2022 21:26:48 +0800 Subject: [PATCH 14/23] fix --- .../connector-jdbc-e2e/pom.xml | 5 +++ .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 31 ++++++++++--------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml index 4ecc1975ce0..716e51e16cb 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml @@ -59,6 +59,11 @@ ${testcontainer.version} test + + org.testcontainers + db2 + ${testcontainer.version} + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index b4692b895a7..b17321300c1 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -17,9 +17,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.Container; -import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Db2Container; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; import java.io.File; import java.io.IOException; @@ -39,10 +40,10 @@ public class JdbcDb2IT extends TestSuiteBase implements TestResource { /** * db2 in dockerhub */ - private static final String IMAGE = "ibmcom/db2:latest"; + private static final String IMAGE = "ibmcom/db2:11.5.0.0a"; private static final String HOST = "e2e_db2"; private static final int PORT = 50000; - private static final int LOCAL_PORT = 50001; + private static final int LOCAL_PORT = 50000; private static final String USER = "DB2INST1"; private static final String PASSWORD = "123456"; private static final String DRIVER = "com.ibm.db2.jcc.DB2Driver"; @@ -51,25 +52,25 @@ public class JdbcDb2IT extends TestSuiteBase implements TestResource { private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE"; private static final String SINK_TABLE = "E2E_TABLE_SINK"; private String jdbcUrl; - private GenericContainer dbServer; + private Db2Container db2; private Connection jdbcConnection; @BeforeAll @Override public void startUp() throws Exception { - dbServer = new GenericContainer<>(IMAGE) + db2 = new Db2Container("ibmcom/db2") + .withExposedPorts(LOCAL_PORT) .withNetwork(NETWORK) .withNetworkAliases(HOST) .withPrivilegedMode(true) + .withDatabaseName(DATABASE) + .withUsername(USER) + .withPassword(PASSWORD) .withLogConsumer(new Slf4jLogConsumer(LOG)) - .withEnv("DB2INST1_PASSWORD", PASSWORD) - .withEnv("DBNAME", DATABASE) - .withEnv("LICENSE", "accept") - .withSharedMemorySize(4 * 1024 * 1024 * 1024L) - ; - dbServer.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); - Startables.deepStart(Stream.of(dbServer)).join(); - jdbcUrl = String.format("jdbc:db2://%s:%s/%s", dbServer.getHost(), LOCAL_PORT, DATABASE); + .acceptLicense(); + Startables.deepStart(Stream.of(db2)).join(); + db2.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); + jdbcUrl = String.format("jdbc:db2://%s:%s/%s", db2.getHost(), LOCAL_PORT, DATABASE); LOG.info("DB2 container started"); given().ignoreExceptions() .await() @@ -83,8 +84,8 @@ public void tearDown() throws Exception { if (jdbcConnection != null) { jdbcConnection.close(); } - if (dbServer != null) { - dbServer.close(); + if (db2 != null) { + db2.close(); } } From 755c8c4c327ba19f1cce349ccdac07fa4948a4ad Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Fri, 21 Oct 2022 21:28:01 +0800 Subject: [PATCH 15/23] fix --- .../apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index b17321300c1..b420e7f7fc6 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -20,7 +20,6 @@ import org.testcontainers.containers.Db2Container; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.lifecycle.Startables; -import org.testcontainers.utility.DockerImageName; import java.io.File; import java.io.IOException; @@ -58,7 +57,7 @@ public class JdbcDb2IT extends TestSuiteBase implements TestResource { @BeforeAll @Override public void startUp() throws Exception { - db2 = new Db2Container("ibmcom/db2") + db2 = new Db2Container(IMAGE) .withExposedPorts(LOCAL_PORT) .withNetwork(NETWORK) .withNetworkAliases(HOST) From acd82fa212f0ec4685579beb6a41b9a0f5a59cd1 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Fri, 21 Oct 2022 23:34:38 +0800 Subject: [PATCH 16/23] fix --- .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 40 +++++++++++-------- .../src/test/resources/init/db2_init.conf | 6 +-- .../resources/jdbc_db2_source_and_sink.conf | 8 ++-- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index b420e7f7fc6..c33ff4261b7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -1,7 +1,6 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc; -import static org.testcontainers.shaded.org.awaitility.Awaitility.given; - +import org.apache.seatunnel.connectors.seatunnel.jdbc.util.JdbcCompareUtil; import org.apache.seatunnel.e2e.common.TestResource; import org.apache.seatunnel.e2e.common.TestSuiteBase; import org.apache.seatunnel.e2e.common.container.TestContainer; @@ -19,7 +18,6 @@ import org.testcontainers.containers.Container; import org.testcontainers.containers.Db2Container; import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.lifecycle.Startables; import java.io.File; import java.io.IOException; @@ -30,8 +28,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; public class JdbcDb2IT extends TestSuiteBase implements TestResource { @@ -39,13 +35,12 @@ public class JdbcDb2IT extends TestSuiteBase implements TestResource { /** * db2 in dockerhub */ - private static final String IMAGE = "ibmcom/db2:11.5.0.0a"; + private static final String IMAGE = "ibmcom/db2"; private static final String HOST = "e2e_db2"; private static final int PORT = 50000; private static final int LOCAL_PORT = 50000; - private static final String USER = "DB2INST1"; + private static final String USER = "db2inst1"; private static final String PASSWORD = "123456"; - private static final String DRIVER = "com.ibm.db2.jcc.DB2Driver"; private static final String DATABASE = "E2E"; private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE"; @@ -58,23 +53,19 @@ public class JdbcDb2IT extends TestSuiteBase implements TestResource { @Override public void startUp() throws Exception { db2 = new Db2Container(IMAGE) - .withExposedPorts(LOCAL_PORT) + .withExposedPorts(PORT) .withNetwork(NETWORK) .withNetworkAliases(HOST) - .withPrivilegedMode(true) .withDatabaseName(DATABASE) .withUsername(USER) .withPassword(PASSWORD) .withLogConsumer(new Slf4jLogConsumer(LOG)) .acceptLicense(); - Startables.deepStart(Stream.of(db2)).join(); db2.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); jdbcUrl = String.format("jdbc:db2://%s:%s/%s", db2.getHost(), LOCAL_PORT, DATABASE); LOG.info("DB2 container started"); - given().ignoreExceptions() - .await() - .atMost(180, TimeUnit.SECONDS) - .untilAsserted(this::initializeJdbcConnection); + db2.start(); + initializeJdbcConnection(); initializeJdbcTable(); } @@ -92,7 +83,7 @@ private void initializeJdbcConnection() throws SQLException, ClassNotFoundExcept Properties properties = new Properties(); properties.setProperty("user", USER); properties.setProperty("password", PASSWORD); - Driver driver = (Driver) Class.forName(DRIVER).newInstance(); + Driver driver = (Driver) Class.forName(db2.getDriverClassName()).newInstance(); jdbcConnection = driver.connect(jdbcUrl, properties); Statement statement = jdbcConnection.createStatement(); ResultSet resultSet = statement.executeQuery("select 1 from SYSSTAT.TABLES"); @@ -145,10 +136,25 @@ void pullImageOK() { @TestTemplate @DisplayName("JDBC-Db2 end to end test") - public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException { + public void testJdbcSourceAndSink(TestContainer container) throws IOException, InterruptedException, SQLException { assertHasData(SOURCE_TABLE); Container.ExecResult execResult = container.executeJob("/jdbc_db2_source_and_sink.conf"); Assertions.assertEquals(0, execResult.getExitCode()); assertHasData(SINK_TABLE); + JdbcCompareUtil.compare(jdbcConnection, String.format("select * from %s.%s", USER, SOURCE_TABLE), + String.format("select * from %s.%s", USER, SINK_TABLE), + "COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC," + + "COL_NUMERIC, COL_NUMBER, COL_REAL, COL_FLOAT,COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR," + + "COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC"); + clearSinkTable(); + } + + + private void clearSinkTable() { + try (Statement statement = jdbcConnection.createStatement()) { + statement.execute(String.format("TRUNCATE TABLE %s.%s", DATABASE, SINK_TABLE)); + } catch (SQLException e) { + throw new RuntimeException("test dm server image error", e); + } } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/init/db2_init.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/init/db2_init.conf index 3274f930622..70092647761 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/init/db2_init.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/init/db2_init.conf @@ -16,7 +16,7 @@ # # db2 sql can't start with \n -table_source = """create table "DB2INST1".E2E_TABLE_SOURCE +table_source = """create table "db2inst1".E2E_TABLE_SOURCE ( COL_BOOLEAN BOOLEAN, COL_INT INT, @@ -57,7 +57,7 @@ table_source = """create table "DB2INST1".E2E_TABLE_SOURCE """ table_sink = """ -create table "DB2INST1".E2E_TABLE_SINK +create table "db2inst1".E2E_TABLE_SINK ( COL_BOOLEAN BOOLEAN, COL_INT INT, @@ -97,7 +97,7 @@ create table "DB2INST1".E2E_TABLE_SINK ) """ -DML = """insert into "DB2INST1".E2E_TABLE_SOURCE +DML = """insert into "db2inst1".E2E_TABLE_SOURCE (COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, COL_NUMERIC, COL_NUMBER, COL_TIMESTAMP, COL_TIME, COL_DATE, COL_REAL, COL_FLOAT, COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf index 6a67a63fa84..cc2e64cfd34 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/resources/jdbc_db2_source_and_sink.conf @@ -25,7 +25,7 @@ source { Jdbc { driver = com.ibm.db2.jcc.DB2Driver url = "jdbc:db2://e2e_db2:50000/E2E" - user = "DB2INST1" + user = "db2inst1" password = "123456" query = """ select COL_BOOLEAN, @@ -48,7 +48,7 @@ source { COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC - from "DB2INST1".E2E_TABLE_SOURCE; + from "db2inst1".E2E_TABLE_SOURCE; """ } @@ -66,10 +66,10 @@ sink { Jdbc { driver = com.ibm.db2.jcc.DB2Driver url = "jdbc:db2://e2e_db2:50000/E2E" - user = "DB2INST1" + user = "db2inst1" password = "123456" query = """ - insert into "DB2INST1".E2E_TABLE_SINK(COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, + insert into "db2inst1".E2E_TABLE_SINK(COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, COL_NUMERIC, COL_NUMBER, COL_REAL, COL_FLOAT, COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC) From bc71aeba1b8344823eae5d4b5b4c22b9d8ce00ab Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Fri, 21 Oct 2022 23:45:25 +0800 Subject: [PATCH 17/23] fix --- .../seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index c33ff4261b7..723a3d8c395 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -121,7 +121,7 @@ private void initializeJdbcTable() { private void assertHasData(String table) { try (Statement statement = jdbcConnection.createStatement()) { - String sql = String.format("select * from \"%s\".%s", USER, table); + String sql = String.format("select * from \"%s\".%s", db2.getUsername(), table); ResultSet source = statement.executeQuery(sql); Assertions.assertTrue(source.next(), "result is null when sql is " + sql); } catch (SQLException e) { @@ -141,8 +141,8 @@ public void testJdbcSourceAndSink(TestContainer container) throws IOException, I Container.ExecResult execResult = container.executeJob("/jdbc_db2_source_and_sink.conf"); Assertions.assertEquals(0, execResult.getExitCode()); assertHasData(SINK_TABLE); - JdbcCompareUtil.compare(jdbcConnection, String.format("select * from %s.%s", USER, SOURCE_TABLE), - String.format("select * from %s.%s", USER, SINK_TABLE), + JdbcCompareUtil.compare(jdbcConnection, String.format("select * from \"%s\".%s", db2.getUsername(), SOURCE_TABLE), + String.format("select * from \"%s\".%s", db2.getUsername(), SINK_TABLE), "COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC," + "COL_NUMERIC, COL_NUMBER, COL_REAL, COL_FLOAT,COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR," + "COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC"); From 4c430d6b9f0595a79ae56163b0a6677e5a3149d9 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sat, 22 Oct 2022 00:56:42 +0800 Subject: [PATCH 18/23] fix --- .../jdbc/internal/dialect/db2/DB2TypeMapper.java | 2 +- .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java index a841254f81f..b67ec4867f5 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2TypeMapper.java @@ -138,7 +138,7 @@ public SeaTunnelDataType mapping(ResultSetMetaData metadata, int colIndex) th default: final String jdbcColumnName = metadata.getColumnName(colIndex); throw new UnsupportedOperationException( - String.format("Doesn't support DM2 type '%s' on column '%s' yet.", columnType, jdbcColumnName)); + String.format("Doesn't support DB2 type '%s' on column '%s' yet.", columnType, jdbcColumnName)); } } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index 723a3d8c395..9474dd9a61f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -121,7 +121,7 @@ private void initializeJdbcTable() { private void assertHasData(String table) { try (Statement statement = jdbcConnection.createStatement()) { - String sql = String.format("select * from \"%s\".%s", db2.getUsername(), table); + String sql = String.format("select * from \"%s\".%s", USER, table); ResultSet source = statement.executeQuery(sql); Assertions.assertTrue(source.next(), "result is null when sql is " + sql); } catch (SQLException e) { @@ -141,8 +141,8 @@ public void testJdbcSourceAndSink(TestContainer container) throws IOException, I Container.ExecResult execResult = container.executeJob("/jdbc_db2_source_and_sink.conf"); Assertions.assertEquals(0, execResult.getExitCode()); assertHasData(SINK_TABLE); - JdbcCompareUtil.compare(jdbcConnection, String.format("select * from \"%s\".%s", db2.getUsername(), SOURCE_TABLE), - String.format("select * from \"%s\".%s", db2.getUsername(), SINK_TABLE), + JdbcCompareUtil.compare(jdbcConnection, String.format("select * from \"%s\".%s", USER, SOURCE_TABLE), + String.format("select * from \"%s\".%s", USER, SINK_TABLE), "COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC," + "COL_NUMERIC, COL_NUMBER, COL_REAL, COL_FLOAT,COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR," + "COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC"); @@ -152,9 +152,10 @@ public void testJdbcSourceAndSink(TestContainer container) throws IOException, I private void clearSinkTable() { try (Statement statement = jdbcConnection.createStatement()) { - statement.execute(String.format("TRUNCATE TABLE %s.%s", DATABASE, SINK_TABLE)); + String truncate = String.format("delete from \"%s\".%s where 1=1;", USER, SINK_TABLE); + statement.execute(truncate); } catch (SQLException e) { - throw new RuntimeException("test dm server image error", e); + throw new RuntimeException("test db2 server image error", e); } } } From bcb270d9732b6c90d0fe940c6bee12a13fafc3a1 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sat, 22 Oct 2022 01:20:01 +0800 Subject: [PATCH 19/23] fix doc --- docs/en/connector-v2/sink/Jdbc.md | 2 +- docs/en/connector-v2/source/Jdbc.md | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/en/connector-v2/sink/Jdbc.md b/docs/en/connector-v2/sink/Jdbc.md index 9f4bfc6d8fd..336c6abf656 100644 --- a/docs/en/connector-v2/sink/Jdbc.md +++ b/docs/en/connector-v2/sink/Jdbc.md @@ -167,4 +167,4 @@ jdbc { - [Feature] Support SQL Server JDBC Source ([2646](https://github.com/apache/incubator-seatunnel/pull/2646)) - [Feature] Support Oracle JDBC Source ([2550](https://github.com/apache/incubator-seatunnel/pull/2550)) - [Feature] Support StarRocks JDBC Source ([3060](https://github.com/apache/incubator-seatunnel/pull/3060)) -- [Feature] Support DB2 JDBC Source ([2410](https://github.com/apache/incubator-seatunnel/pull/2410)) \ No newline at end of file +- [Feature] Support DB2 JDBC Sink ([2410](https://github.com/apache/incubator-seatunnel/pull/2410)) \ No newline at end of file diff --git a/docs/en/connector-v2/source/Jdbc.md b/docs/en/connector-v2/source/Jdbc.md index 0a6293b82bf..33cd324be76 100644 --- a/docs/en/connector-v2/source/Jdbc.md +++ b/docs/en/connector-v2/source/Jdbc.md @@ -90,17 +90,17 @@ in parallel according to the concurrency of tasks. there are some reference value for params above. -| datasource | driver | url | maven | -| ---------- | -------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -| mysql | com.mysql.cj.jdbc.Driver | jdbc:mysql://localhost:3306/test | https://mvnrepository.com/artifact/mysql/mysql-connector-java | -| postgresql | org.postgresql.Driver | jdbc:postgresql://localhost:5432/postgres | https://mvnrepository.com/artifact/org.postgresql/postgresql | -| dm | dm.jdbc.driver.DmDriver | jdbc:dm://localhost:5236 | https://mvnrepository.com/artifact/com.dameng/DmJdbcDriver18 | -| phoenix | org.apache.phoenix.queryserver.client.Driver | jdbc:phoenix:thin:url=http://localhost:8765;serialization=PROTOBUF | https://mvnrepository.com/artifact/com.aliyun.phoenix/ali-phoenix-shaded-thin-client | -| sqlserver | com.microsoft.sqlserver.jdbc.SQLServerDriver | jdbc:microsoft:sqlserver://localhost:1433 | https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc | -| oracle | oracle.jdbc.OracleDriver | jdbc:oracle:thin:@localhost:1521/xepdb1 | https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8 | -| gbase8a | com.gbase.jdbc.Driver | jdbc:gbase://e2e_gbase8aDb:5258/test | https://www.gbase8.cn/wp-content/uploads/2020/10/gbase-connector-java-8.3.81.53-build55.5.7-bin_min_mix.jar | -| starrocks | com.mysql.cj.jdbc.Driver | jdbc:mysql://localhost:3306/test | https://mvnrepository.com/artifact/mysql/mysql-connector-java | -| db2 | com.ibm.db2.jcc.DB2Driver | jdbc:db2://localhost:50000/testdb | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc/db2jcc4 | +| datasource | driver | url | maven | +|------------|----------------------------------------------|--------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------| +| mysql | com.mysql.cj.jdbc.Driver | jdbc:mysql://localhost:3306/test | https://mvnrepository.com/artifact/mysql/mysql-connector-java | +| postgresql | org.postgresql.Driver | jdbc:postgresql://localhost:5432/postgres | https://mvnrepository.com/artifact/org.postgresql/postgresql | +| dm | dm.jdbc.driver.DmDriver | jdbc:dm://localhost:5236 | https://mvnrepository.com/artifact/com.dameng/DmJdbcDriver18 | +| phoenix | org.apache.phoenix.queryserver.client.Driver | jdbc:phoenix:thin:url=http://localhost:8765;serialization=PROTOBUF | https://mvnrepository.com/artifact/com.aliyun.phoenix/ali-phoenix-shaded-thin-client | +| sqlserver | com.microsoft.sqlserver.jdbc.SQLServerDriver | jdbc:microsoft:sqlserver://localhost:1433 | https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc | +| oracle | oracle.jdbc.OracleDriver | jdbc:oracle:thin:@localhost:1521/xepdb1 | https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8 | +| gbase8a | com.gbase.jdbc.Driver | jdbc:gbase://e2e_gbase8aDb:5258/test | https://www.gbase8.cn/wp-content/uploads/2020/10/gbase-connector-java-8.3.81.53-build55.5.7-bin_min_mix.jar | +| starrocks | com.mysql.cj.jdbc.Driver | jdbc:mysql://localhost:3306/test | https://mvnrepository.com/artifact/mysql/mysql-connector-java | +| db2 | com.ibm.db2.jcc.DB2Driver | jdbc:db2://localhost:50000/testdb | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc/db2jcc4 | ## Example @@ -144,3 +144,4 @@ parallel: - [Feature] Support Oracle JDBC Source ([2550](https://github.com/apache/incubator-seatunnel/pull/2550)) - [Feature] Support StarRocks JDBC Source ([3060](https://github.com/apache/incubator-seatunnel/pull/3060)) - [Feature] Support GBase8a JDBC Source ([3026](https://github.com/apache/incubator-seatunnel/pull/3026)) +- [Feature] Support DB2 JDBC Source ([2410](https://github.com/apache/incubator-seatunnel/pull/2410)) From 8b431837cc4f693fef438fd7d8eb87c82840226b Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sat, 22 Oct 2022 01:28:04 +0800 Subject: [PATCH 20/23] fix style --- .../seatunnel-spark-connector-v2-example/pom.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml b/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml index bef93603017..23f22f062ac 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml @@ -99,11 +99,6 @@ lz4 1.3.0 - - org.apache.seatunnel - connector-jdbc - 2.1.3-SNAPSHOT - - + \ No newline at end of file From 3a7371ff91c04c0992d1a6e383bc1d8bb0f328bd Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sat, 22 Oct 2022 01:33:25 +0800 Subject: [PATCH 21/23] fix style --- .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index 9474dd9a61f..8b52c63ccb2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.connectors.seatunnel.jdbc; import org.apache.seatunnel.connectors.seatunnel.jdbc.util.JdbcCompareUtil; @@ -149,7 +166,6 @@ public void testJdbcSourceAndSink(TestContainer container) throws IOException, I clearSinkTable(); } - private void clearSinkTable() { try (Statement statement = jdbcConnection.createStatement()) { String truncate = String.format("delete from \"%s\".%s where 1=1;", USER, SINK_TABLE); From 67076178f615bc01b0cc9334e0f0f5eff5da3d38 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sat, 22 Oct 2022 01:43:30 +0800 Subject: [PATCH 22/23] fix driver jar --- .../seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index 8b52c63ccb2..80cea8c02be 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -20,7 +20,9 @@ import org.apache.seatunnel.connectors.seatunnel.jdbc.util.JdbcCompareUtil; import org.apache.seatunnel.e2e.common.TestResource; import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory; import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.seatunnel.e2e.common.junit.TestContainerExtension; import com.google.common.collect.Lists; import com.typesafe.config.Config; @@ -58,6 +60,7 @@ public class JdbcDb2IT extends TestSuiteBase implements TestResource { private static final int LOCAL_PORT = 50000; private static final String USER = "db2inst1"; private static final String PASSWORD = "123456"; + public static final String DB2_DRIVER_JAR = "https://repo1.maven.org/maven2/com/ibm/db2/jcc/db2jcc/db2jcc4/db2jcc-db2jcc4.jar"; private static final String DATABASE = "E2E"; private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE"; @@ -66,6 +69,12 @@ public class JdbcDb2IT extends TestSuiteBase implements TestResource { private Db2Container db2; private Connection jdbcConnection; + @TestContainerExtension + private final ContainerExtendedFactory extendedFactory = container -> { + Container.ExecResult extraCommands = container.execInContainer("bash", "-c", "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && curl -O " + DB2_DRIVER_JAR); + Assertions.assertEquals(0, extraCommands.getExitCode()); + }; + @BeforeAll @Override public void startUp() throws Exception { From ea5c7e7247c93c75919fd2dcb115e4306fc5c0b0 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Wed, 26 Oct 2022 23:06:29 +0800 Subject: [PATCH 23/23] fix some code --- .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 3 +- .../connector-jdbc-spark-e2e/pom.xml | 7 -- .../examples/jdbc_db2_source_and_sink.conf | 90 ------------------- 3 files changed, 2 insertions(+), 98 deletions(-) delete mode 100644 seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/jdbc_db2_source_and_sink.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index 80cea8c02be..12159ea97ef 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -37,6 +37,7 @@ import org.testcontainers.containers.Container; import org.testcontainers.containers.Db2Container; import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.utility.DockerLoggerFactory; import java.io.File; import java.io.IOException; @@ -85,7 +86,7 @@ public void startUp() throws Exception { .withDatabaseName(DATABASE) .withUsername(USER) .withPassword(PASSWORD) - .withLogConsumer(new Slf4jLogConsumer(LOG)) + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) .acceptLicense(); db2.setPortBindings(Lists.newArrayList(String.format("%s:%s", LOCAL_PORT, PORT))); jdbcUrl = String.format("jdbc:db2://%s:%s/%s", db2.getHost(), LOCAL_PORT, DATABASE); diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml index a0cb7eeadc3..ee7629f1498 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-jdbc-spark-e2e/pom.xml @@ -96,13 +96,6 @@ com.microsoft.sqlserver mssql-jdbc - - - org.testcontainers - db2 - 1.17.3 - test - diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/jdbc_db2_source_and_sink.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/jdbc_db2_source_and_sink.conf deleted file mode 100644 index ac42d635f41..00000000000 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/jdbc_db2_source_and_sink.conf +++ /dev/null @@ -1,90 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### - -env { - # You can set spark configuration here - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local - job.mode = "BATCH" -} - -source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** - Jdbc { - driver = com.ibm.db2.jcc.DB2Driver - url = "jdbc:db2://localhost:50000/testdb" - user = DB2INST1 - password = 123456 - query = """ - select COL_BOOLEAN, - COL_INT, - COL_INTEGER, - COL_SMALLINT, - COL_BIGINT, - COL_DECIMAL, - COL_DEC, - COL_NUMERIC, - COL_NUMBER, - COL_REAL, - COL_FLOAT, - COL_DOUBLE_PRECISION, - COL_DOUBLE, - COL_DECFLOAT, - COL_CHAR, - COL_VARCHAR, - COL_LONG_VARCHAR, - COL_GRAPHIC, - COL_VARGRAPHIC, - COL_LONG_VARGRAPHIC - from "DB2INST1".E2E_TABLE_SOURCE; - """ - } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc -} - -transform { - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql -} - -sink { - Jdbc { - driver = com.ibm.db2.jcc.DB2Driver - url = "jdbc:db2://localhost:50000/testdb" - user = DB2INST1 - password = 123456 - query = """ - insert into "DB2INST1".E2E_TABLE_SINK(COL_BOOLEAN, COL_INT, COL_INTEGER, COL_SMALLINT, COL_BIGINT, COL_DECIMAL, COL_DEC, - COL_NUMERIC, COL_NUMBER, COL_REAL, COL_FLOAT, - COL_DOUBLE_PRECISION, COL_DOUBLE, COL_DECFLOAT, COL_CHAR, COL_VARCHAR, - COL_LONG_VARCHAR, COL_GRAPHIC, COL_VARGRAPHIC, COL_LONG_VARGRAPHIC) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -""" - } - - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc -}