[Improve][Connector-V2][iceberg] Refactor iceberg connector e2e test …

…cases (apache#3820) * [Improve][Connector-V2][ioTdb] Refactor iceberg connector e2e test cases * [Improve][Connector-V2][ioTdb] Refactor iceberg connector e2e test cases * [Improve][Connector-V2][ioTdb] Refactor iceberg connector e2e test cases * [Improve][Connector-V2][iceberg] Refactor iceberg connector e2e test cases * [Improve][Connector-V2][iceberg] Refactor iceberg connector e2e test cases Co-authored-by: zhouyao <[email protected]>
lhyundeadsoul · Jan 3, 2023 · eff8ef4 · eff8ef4
1 parent fe2bca9
commit eff8ef4
Show file tree

Hide file tree

Showing 17 changed files with 71 additions and 699 deletions.
diff --git a/...2-e2e/connector-iceberg-flink-e2e/pom.xml → ...ctor-v2-e2e/connector-iceberg-e2e/pom.xml b/...2-e2e/connector-iceberg-flink-e2e/pom.xml → ...ctor-v2-e2e/connector-iceberg-e2e/pom.xml
@@ -18,25 +18,29 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <groupId>org.apache.seatunnel</groupId>
-        <artifactId>seatunnel-flink-connector-v2-e2e</artifactId>
+        <artifactId>seatunnel-connector-v2-e2e</artifactId>
         <version>${revision}</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>connector-iceberg-flink-e2e</artifactId>
+    <artifactId>connector-iceberg-e2e</artifactId>
 
     <dependencies>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
-            <artifactId>connector-flink-e2e-base</artifactId>
+            <artifactId>connector-iceberg</artifactId>
             <version>${project.version}</version>
-            <classifier>tests</classifier>
-            <type>test-jar</type>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
-            <artifactId>connector-iceberg</artifactId>
+            <artifactId>connector-assert</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-console</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>

diff --git a/...k/v2/icegerg/hadoop3/IcebergSourceIT.java → ...2e/connector/iceberg/IcebergSourceIT.java b/...k/v2/icegerg/hadoop3/IcebergSourceIT.java → ...2e/connector/iceberg/IcebergSourceIT.java
@@ -15,13 +15,17 @@
  * limitations under the License.
  */
 
-package org.apache.seatunnel.e2e.flink.v2.icegerg.hadoop3;
+package org.apache.seatunnel.e2e.connector.iceberg;
 
 import static org.apache.seatunnel.connectors.seatunnel.iceberg.config.IcebergCatalogType.HADOOP;
 
 import org.apache.seatunnel.connectors.seatunnel.iceberg.IcebergCatalogFactory;
 import org.apache.seatunnel.connectors.seatunnel.iceberg.config.IcebergCatalogType;
-import org.apache.seatunnel.e2e.flink.FlinkContainer;
+import org.apache.seatunnel.e2e.common.TestResource;
+import org.apache.seatunnel.e2e.common.TestSuiteBase;
+import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
+import org.apache.seatunnel.e2e.common.container.TestContainer;
+import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
 
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hadoop.conf.Configuration;
@@ -42,9 +46,10 @@
 import org.apache.iceberg.io.FileAppender;
 import org.apache.iceberg.io.FileAppenderFactory;
 import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
 import org.testcontainers.containers.Container;
 import org.testcontainers.utility.MountableFile;
 
@@ -61,7 +66,7 @@
 import java.util.List;
 
 @Slf4j
-public class IcebergSourceIT extends FlinkContainer {
+public class IcebergSourceIT extends TestSuiteBase implements TestResource {
 
     private static final TableIdentifier TABLE = TableIdentifier.of(
         Namespace.of("database1"), "source");
@@ -90,22 +95,31 @@ public class IcebergSourceIT extends FlinkContainer {
 
     private static final String CATALOG_NAME = "seatunnel";
     private static final IcebergCatalogType CATALOG_TYPE = HADOOP;
-    private static final String CATALOG_DIR = "/tmp/seatunnel/iceberg/flink/";
+    private static final String CATALOG_DIR = "/tmp/seatunnel/iceberg/hadoop/";
     private static final String WAREHOUSE = "file://" + CATALOG_DIR;
     private static Catalog CATALOG;
 
+    @TestContainerExtension
+    private final ContainerExtendedFactory extendedFactory = container -> {
+        container.copyFileToContainer(MountableFile.forHostPath(CATALOG_DIR), CATALOG_DIR);
+    };
+
     @BeforeEach
-    public void start() {
+    @Override
+    public void startUp() throws Exception {
         initializeIcebergTable();
         batchInsertData();
-        MountableFile catalogPath = MountableFile.forHostPath(CATALOG_DIR);
-        jobManager.copyFileToContainer(catalogPath, CATALOG_DIR);
-        taskManager.copyFileToContainer(catalogPath, CATALOG_DIR);
     }
 
-    @Test
-    public void testIcebergSource() throws IOException, InterruptedException {
-        Container.ExecResult execResult = executeSeaTunnelFlinkJob("/iceberg/iceberg_source.conf");
+    @AfterAll
+    @Override
+    public void tearDown() throws Exception {
+
+    }
+
+    @TestTemplate
+    public void testIcebergSource(TestContainer container) throws IOException, InterruptedException {
+        Container.ExecResult execResult = container.executeJob("/iceberg/iceberg_source.conf");
         Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
     }
 

diff --git a/...est/resources/iceberg/iceberg_source.conf → ...est/resources/iceberg/iceberg_source.conf b/...est/resources/iceberg/iceberg_source.conf → ...est/resources/iceberg/iceberg_source.conf
@@ -34,7 +34,6 @@ source {
       f5 = "float"
       f6 = "double"
       f7 = "date"
-      f8 = "time"
       f9 = "timestamp"
       f10 = "timestamp"
       f11 = "string"
@@ -46,7 +45,7 @@ source {
     }
     catalog_name = "seatunnel"
     catalog_type = "hadoop"
-    warehouse = "file:///tmp/seatunnel/iceberg/flink/"
+    warehouse = "file:///tmp/seatunnel/iceberg/hadoop/"
     namespace = "database1"
     table = "source"
   }

diff --git a/...2-e2e/connector-iceberg-spark-e2e/pom.xml → ...e2e/connector-iceberg-hadoop3-e2e/pom.xml b/...2-e2e/connector-iceberg-spark-e2e/pom.xml → ...e2e/connector-iceberg-hadoop3-e2e/pom.xml
@@ -18,47 +18,36 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <groupId>org.apache.seatunnel</groupId>
-        <artifactId>seatunnel-spark-connector-v2-e2e</artifactId>
+        <artifactId>seatunnel-connector-v2-e2e</artifactId>
         <version>${revision}</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>connector-iceberg-spark-e2e</artifactId>
+    <artifactId>connector-iceberg-hadoop3-e2e</artifactId>
 
     <properties>
         <hadoop-client.version>3.3.4</hadoop-client.version>
     </properties>
 
     <dependencies>
-        <dependency>
-            <groupId>org.apache.seatunnel</groupId>
-            <artifactId>connector-spark-e2e-base</artifactId>
-            <version>${project.version}</version>
-            <classifier>tests</classifier>
-            <type>test-jar</type>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-iceberg</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
-            <artifactId>connector-console</artifactId>
+            <artifactId>connector-assert</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
-            <artifactId>connector-assert</artifactId>
+            <artifactId>connector-console</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
@@ -72,5 +61,4 @@
             </exclusions>
         </dependency>
     </dependencies>
-
 </project>
diff --git a/...e2e/flink/v2/iceberg/IcebergSourceIT.java → ...ctor/iceberg/hadoop3/IcebergSourceIT.java b/...e2e/flink/v2/iceberg/IcebergSourceIT.java → ...ctor/iceberg/hadoop3/IcebergSourceIT.java
@@ -15,13 +15,17 @@
  * limitations under the License.
  */
 
-package org.apache.seatunnel.e2e.flink.v2.iceberg;
+package org.apache.seatunnel.e2e.connector.iceberg.hadoop3;
 
 import static org.apache.seatunnel.connectors.seatunnel.iceberg.config.IcebergCatalogType.HADOOP;
 
 import org.apache.seatunnel.connectors.seatunnel.iceberg.IcebergCatalogFactory;
 import org.apache.seatunnel.connectors.seatunnel.iceberg.config.IcebergCatalogType;
-import org.apache.seatunnel.e2e.flink.FlinkContainer;
+import org.apache.seatunnel.e2e.common.TestResource;
+import org.apache.seatunnel.e2e.common.TestSuiteBase;
+import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
+import org.apache.seatunnel.e2e.common.container.TestContainer;
+import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
 
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hadoop.conf.Configuration;
@@ -42,9 +46,10 @@
 import org.apache.iceberg.io.FileAppender;
 import org.apache.iceberg.io.FileAppenderFactory;
 import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
 import org.testcontainers.containers.Container;
 import org.testcontainers.utility.MountableFile;
 
@@ -61,7 +66,7 @@
 import java.util.List;
 
 @Slf4j
-public class IcebergSourceIT extends FlinkContainer {
+public class IcebergSourceIT extends TestSuiteBase implements TestResource {
 
     private static final TableIdentifier TABLE = TableIdentifier.of(
         Namespace.of("database1"), "source");
@@ -90,22 +95,31 @@ public class IcebergSourceIT extends FlinkContainer {
 
     private static final String CATALOG_NAME = "seatunnel";
     private static final IcebergCatalogType CATALOG_TYPE = HADOOP;
-    private static final String CATALOG_DIR = "/tmp/seatunnel/iceberg/flink/";
+    private static final String CATALOG_DIR = "/tmp/seatunnel/iceberg/hadoop3/";
     private static final String WAREHOUSE = "file://" + CATALOG_DIR;
     private static Catalog CATALOG;
 
+    @TestContainerExtension
+    private final ContainerExtendedFactory extendedFactory = container -> {
+        container.copyFileToContainer(MountableFile.forHostPath(CATALOG_DIR), CATALOG_DIR);
+    };
+
     @BeforeEach
-    public void start() {
+    @Override
+    public void startUp() throws Exception {
         initializeIcebergTable();
         batchInsertData();
-        MountableFile catalogPath = MountableFile.forHostPath(CATALOG_DIR);
-        jobManager.copyFileToContainer(catalogPath, CATALOG_DIR);
-        taskManager.copyFileToContainer(catalogPath, CATALOG_DIR);
     }
 
-    @Test
-    public void testIcebergSource() throws IOException, InterruptedException {
-        Container.ExecResult execResult = executeSeaTunnelFlinkJob("/iceberg/iceberg_source.conf");
+    @AfterAll
+    @Override
+    public void tearDown() throws Exception {
+
+    }
+
+    @TestTemplate
+    public void testIcebergSource(TestContainer container) throws IOException, InterruptedException {
+        Container.ExecResult execResult = container.executeJob("/iceberg/iceberg_source.conf");
         Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
     }
 

diff --git a/...est/resources/iceberg/iceberg_source.conf → ...est/resources/iceberg/iceberg_source.conf b/...est/resources/iceberg/iceberg_source.conf → ...est/resources/iceberg/iceberg_source.conf
@@ -48,7 +48,7 @@ source {
         }
         catalog_name = "seatunnel"
         catalog_type = "hadoop"
-        warehouse = "file:///tmp/seatunnel/iceberg/spark/"
+        warehouse = "file:///tmp/seatunnel/iceberg/hadoop3/"
         namespace = "database1"
         table = "source"
     }

diff --git a/...k-e2e/src/test/resources/log4j.properties → ...3-e2e/src/test/resources/log4j.properties b/...k-e2e/src/test/resources/log4j.properties → ...3-e2e/src/test/resources/log4j.properties
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml
@@ -43,6 +43,9 @@
         <module>connector-elasticsearch-e2e</module>
         <module>connector-iotdb-e2e</module>
         <module>connector-cdc-mysql-e2e</module>
+        <module>connector-tdengine-e2e</module>
+        <module>connector-iceberg-e2e</module>
+        <module>connector-iceberg-hadoop3-e2e</module>
     </modules>
 
     <artifactId>seatunnel-connector-v2-e2e</artifactId>

diff --git a/...nnector-v2-e2e/connector-iceberg-flink-e2e/src/test/resources/iceberg/iceberg_source.conf b/...nnector-v2-e2e/connector-iceberg-flink-e2e/src/test/resources/iceberg/iceberg_source.conf