[Improve][CI] split it to 4 github tasks (apache#3832)

* split it to 4 github tasks
lhyundeadsoul · Jan 3, 2023 · eb5b844 · eb5b844
1 parent 556f1cd
commit eb5b844
Show file tree

Hide file tree

Showing 4 changed files with 127 additions and 8 deletions.
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -315,9 +315,9 @@ jobs:
         env:
           MAVEN_OPTS: -Xmx2048m
 
-  integration-test:
+  updated-modules-integration-test:
     needs: [ changes, sanity-check ]
-    if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '')
+    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -332,17 +332,120 @@ jobs:
           java-version: ${{ matrix.java }}
           distribution: 'temurin'
           cache: 'maven'
-      - name: run all modules integration test
+      - name: run updated modules integration test
+        if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
+        run: |
+          ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -Pci
+        env:
+          MAVEN_OPTS: -Xmx2048m
+
+  engine-and-transform-v2-it:
+    needs: [ changes, sanity-check ]
+    if: needs.changes.outputs.api == 'true'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        java: [ '8', '11' ]
+        os: [ 'ubuntu-latest' ]
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: run some modules integration test
         if: needs.changes.outputs.api == 'true'
         run: |
-          ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
+          ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e,:connector-seatunnel-e2e-base -am -Pci
         env:
           MAVEN_OPTS: -Xmx2048m
 
-      - name: run updated modules integration test
-        if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
+  all-connectors-it-1:
+    needs: [ changes, sanity-check ]
+    if: needs.changes.outputs.api == 'true'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        java: [ '8', '11' ]
+        os: [ 'ubuntu-latest' ]
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: run some modules integration test
+        if: needs.changes.outputs.api == 'true'
         run: |
-          ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -Pci
+          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-flink-connector-v2-e2e > /tmp/sub_module.txt
+          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-spark-connector-v2-e2e >> /tmp/sub_module.txt
+          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
+          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
+          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 3 0`
+          ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
         env:
           MAVEN_OPTS: -Xmx2048m
 
+  all-connectors-it-2:
+    needs: [ changes, sanity-check ]
+    if: needs.changes.outputs.api == 'true'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        java: [ '8', '11' ]
+        os: [ 'ubuntu-latest' ]
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: run some modules integration test
+        if: needs.changes.outputs.api == 'true'
+        run: |
+          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-flink-connector-v2-e2e > /tmp/sub_module.txt
+          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-spark-connector-v2-e2e >> /tmp/sub_module.txt
+          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
+          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
+          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 3 1`
+          ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
+        env:
+          MAVEN_OPTS: -Xmx2048m
+
+  all-connectors-it-3:
+    needs: [ changes, sanity-check ]
+    if: needs.changes.outputs.api == 'true'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        java: [ '8', '11' ]
+        os: [ 'ubuntu-latest' ]
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: run some modules integration test
+        if: needs.changes.outputs.api == 'true'
+        run: |
+          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-flink-connector-v2-e2e > /tmp/sub_module.txt
+          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-spark-connector-v2-e2e >> /tmp/sub_module.txt
+          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
+          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
+          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 3 2`
+          ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
+        env:
+          MAVEN_OPTS: -Xmx2048m
diff --git a/seatunnel-e2e/seatunnel-e2e-common/pom.xml b/seatunnel-e2e/seatunnel-e2e-common/pom.xml
@@ -31,6 +31,12 @@
             <artifactId>seatunnel-api</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-transforms-v2</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-transforms-v2-e2e/pom.xml
@@ -51,7 +51,6 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-fake</artifactId>

diff --git a/tools/update_modules_check/update_modules_check.py b/tools/update_modules_check/update_modules_check.py
@@ -148,6 +148,15 @@ def get_deleted_modules(files):
     output_module = output_module[1:len(output_module)]
     print(output_module)
 
+def get_sub_it_modules(modules, total_num, current_num):
+    modules_arr = modules.split(",")
+    output = ""
+    for i,module in enumerate(modules_arr):
+        if len(module) > 0 and i % int(total_num) == int(current_num):
+            output = output + ",:" + module
+
+    output = output[1:len(output)]
+    print(output)
 
 def main(argv):
     if argv[1] == "cv2":
@@ -176,6 +185,8 @@ def main(argv):
         get_deleted_modules(argv[2])
     elif argv[1] == "rm":
         remove_deleted_modules(argv[2], argv[3])
+    elif argv[1] == "sub_it_module":
+        get_sub_it_modules(argv[2], argv[3], argv[4])
 
 
 if __name__ == "__main__":