integration_test_mysql.yaml

name: Integration-Mysql

on:
  push:
    branches:
      - master
      - "release-[0-9].[0-9]*"
    paths-ignore:
      - '**/*.md'
      - '**/OWNERS'
      - 'OWNERS'
      - 'OWNERS_ALIASES'

  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    branches:
      - master
      - "release-[0-9].[0-9]*"
    paths-ignore:
      - '**/*.md'
      - '**/OWNERS'
      - 'OWNERS'
      - 'OWNERS_ALIASES'

# See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
concurrency:
  group: ${{ github.ref }}-${{ github.workflow }}
  cancel-in-progress: true

jobs:
  # # To boost the test speed, we split every 15 test cases into a group.
  e2e_test_group_1:
    ## Only run ci when PR is not draft
    if: github.event.pull_request.draft == false

    runs-on: ubuntu-latest 
    name: E2E Test Group 1
    steps:
      - name: Check out code
        uses: actions/checkout@v2

      - name: Setup Go environment
        uses: actions/setup-go@v3
        with:
          go-version: '1.23'

      - name: Integration Build
        run: |
          tests/scripts/download-integration-test-binaries.sh master true
          go build -o ./tools/bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl 
          make integration_test_build
          ls -l bin/ && ls -l tools/bin/

      - name: Test charset_gbk
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=charset_gbk 

      - name: Test changefeed_finish
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=changefeed_finish
      
      - name: Test sql_mode
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=sql_mode
        
      - name: Test changefeed_pause_resume
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=changefeed_pause_resume

      - name: Test changefeed_reconstruct
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=changefeed_reconstruct

      - name: Test common_1
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=common_1

      - name: Test foreign_key
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=foreign_key

      # The 7th case in this group
      - name: Test generate_column
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=generate_column
      
      # The 8th case in this group
      - name: Test many_pk_or_uk
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=many_pk_or_uk
      
      # The 9th case in this group
      - name: Test drop_many_tables
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=drop_many_tables

      # The 10th case in this group     
      - name: Test new_ci_collation
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=new_ci_collation

      # The 11th case in this group     
      - name: Test region_merge
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=region_merge

      # The 12th case in this group     
      - name: Test safe mode
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=safe_mode

      # The 13th case in this group     
      - name: Test savepoint
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=savepoint

      - name: Test server config compatibility
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=server_config_compatibility

      - name: Test split region
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=split_region
      
      # The 16th case in this group     
      - name: Test changefeed resume with checkpoint ts
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=changefeed_resume_with_checkpoint_ts  

      - name: Test capture suicide while balance table
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=capture_suicide_while_balance_table 

      - name: Test kv client stream reconnect
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=kv_client_stream_reconnect  

      - name: Upload test logs
        if: always()
        uses: ./.github/actions/upload-test-logs
        with:
          log-name: e2e_test_group_1

  # # To boost the test speed, we split every 15 test cases into a group.
  e2e_test_group_1_on_7-5:
    ## Only run ci when PR is not draft
    if: github.event.pull_request.draft == false

    runs-on: ubuntu-latest 
    name: E2E Test Group 1 on 7.5
    steps:
      - name: Check out code
        uses: actions/checkout@v2

      - name: Setup Go environment
        uses: actions/setup-go@v3
        with:
          go-version: '1.23'

      - name: Integration Build
        run: |
          tests/scripts/download-integration-test-binaries.sh master true v7.5.5
          go build -o ./tools/bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl 
          make integration_test_build
          ls -l bin/ && ls -l tools/bin/

      - name: Test charset_gbk
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=charset_gbk 

      - name: Test changefeed_finish
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=changefeed_finish
      
      # - name: Test sql_mode
      #   if: ${{ success() }}
      #   run: |
      #     export TICDC_NEWARCH=true && make integration_test CASE=sql_mode
        
      - name: Test changefeed_pause_resume
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=changefeed_pause_resume

      - name: Test changefeed_reconstruct
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=changefeed_reconstruct

      - name: Test common_1
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=common_1

      - name: Test foreign_key
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=foreign_key

      # The 7th case in this group
      - name: Test generate_column
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=generate_column
      
      # The 8th case in this group
      - name: Test many_pk_or_uk
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=many_pk_or_uk
      
      # The 9th case in this group
      - name: Test drop_many_tables
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=drop_many_tables

      # The 10th case in this group     
      - name: Test new_ci_collation
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=new_ci_collation

      # The 11th case in this group     
      - name: Test region_merge
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=region_merge

      # The 12th case in this group     
      - name: Test safe mode
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=safe_mode

      # The 13th case in this group     
      - name: Test savepoint
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=savepoint

      # The 14th case in this group     
      - name: Test server config compatibility
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=server_config_compatibility

      # The 15th case in this group     
      - name: Test split region
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=split_region

      - name: Upload test logs
        if: always()
        uses: ./.github/actions/upload-test-logs
        with:
          log-name: e2e_test_group_1_on_7.5

  e2e_test_group_2:
    ## Only run ci when PR is not draft
    if: github.event.pull_request.draft == false

    runs-on: ubuntu-latest 
    name: E2E Test Group 2
    steps:
      - name: Check out code
        uses: actions/checkout@v2

      - name: Setup Go environment
        uses: actions/setup-go@v3
        with:
          go-version: '1.23'

      - name: Integration Build
        run: |
          tests/scripts/download-integration-test-binaries.sh master true
          go build -o ./tools/bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl 
          make integration_test_build
          ls -l bin/ && ls -l tools/bin/

      - name: Test api_v2
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=api_v2

      - name: Test autorandom
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=autorandom

      - name: Test availability
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=availability

      - name: Test bank
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=bank

      - name: Test batch_add_table
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=batch_add_table

      - name: Test batch_update_to_no_batch
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=batch_update_to_no_batch

      - name: Test ci_collation_compatibility
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=ci_collation_compatibility

      - name: Test multi_capture
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=multi_capture

      - name: Test multi_cdc_cluster
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=multi_cdc_cluster

      - name: Test multi_rocks
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=multi_rocks

      - name: Test resourcecontrol
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=resourcecontrol

      - name: Test row_format
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=row_format

      - name: Test tiflash
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=tiflash

      # The 14th case in this group     
      - name: Test vector
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=vector

      # The 15th case in this group     
      - name: Test simple
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=simple

      - name: Upload test logs
        if: always()
        uses: ./.github/actions/upload-test-logs
        with:
          log-name: e2e_test_group_2

  e2e_test_group_3:
    ## Only run ci when PR is not draft
    if: github.event.pull_request.draft == false

    runs-on: ubuntu-latest 
    name: E2E Test Group 3
    steps:
      - name: Check out code
        uses: actions/checkout@v2

      - name: Setup Go environment
        uses: actions/setup-go@v3
        with:
          go-version: '1.23'
      
      - name: Set up S3cmd cli tool
        uses: s3-actions/s3cmd@v1.9.0
        with:
          provider: linode 

      - name: Run s3cmd command
        run: s3cmd --version

      - name: Integration Build
        run: |
          tests/scripts/download-integration-test-binaries.sh master true
          go build -o ./tools/bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl 
          make integration_test_build
          ls -l bin/ && ls -l tools/bin/

      # - name: Test http_api
      #   if: ${{ success() }}
      #   run: |
      #     export TICDC_NEWARCH=true && make integration_test CASE=http_api

      - name: Test partition_table
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=partition_table
     
      - name: Test ddl_attributes
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=ddl_attributes
      
      - name: Test http_api_tls
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=http_api_tls

      # - name: Test http_api_tls_with_user_auth
      #   if: ${{ success() }}
      #   run: |
      #     export TICDC_NEWARCH=true && make integration_test CASE=http_api_tls_with_user_auth
      
      - name: Test default_value
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=default_value

      - name: Test http_proxies
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=http_proxies

      - name: Test ddl_reentrant
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=ddl_reentrant

      - name: Test syncpoint
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=syncpoint

      - name: Test force_replicate_table
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=force_replicate_table

      - name: Test tidb_mysql_test
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=tidb_mysql_test


      - name: Test resolve_lock
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=resolve_lock

      - name: Test move_table
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=move_table

      - name: Test changefeed_error
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=changefeed_error

      # The 16th case in this group
      - name: Test capture_session_done_during_task
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=capture_session_done_during_task

      - name: Test changefeed_dup_error_restart
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=changefeed_dup_error_restart

      - name: Test mysql_sink_retry
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=mysql_sink_retry
      
      - name: Test cdc
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=cdc

      - name: Test overwrite_resume_with_syncpoint
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=overwrite_resume_with_syncpoint
      
      - name: Test ddl_for_split_tables
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=ddl_for_split_tables

      - name: Test multi_source
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=multi_source

      # The 21th case in this group
      - name: Test multi_tables_ddl
        if: ${{ success() }}
        run: |
          export TICDC_NEWARCH=true && make integration_test CASE=multi_tables_ddl

      - name: Upload test logs
        if: always()
        uses: ./.github/actions/upload-test-logs
        with:
          log-name: e2e_test_group_3


  failover_e2e_test1:
      ## Only run ci when PR is not draft
    if: github.event.pull_request.draft == false

    runs-on: ubuntu-latest 
    name: Failover E2E Test
    steps:
      - name: Check out code
        uses: actions/checkout@v2

      - name: Setup Go environment
        uses: actions/setup-go@v3
        with:
          go-version: '1.23'

      - name: Integration Build
        run: |
          tests/scripts/download-integration-test-binaries.sh master true
          go build -o ./tools/bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl 
          make integration_test_build
          ls -l bin/ && ls -l tools/bin/

      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 

      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 

      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 

      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 

      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 

      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 
      
      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 

      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 

      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 

      - name: Test fail_over
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over 
      
      - name: Test fail_over_ddl_A
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_A
      
      - name: Test fail_over_ddl_B
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_B
      
      - name: Test fail_over_ddl_C
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_C
      
      - name: Test fail_over_ddl_D
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_D

      - name: Test fail_over_ddl_E
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_E
      
      - name: Test fail_over_ddl_F
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_F
      
      - name: Test fail_over_ddl_G
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_G
      
      - name: Test fail_over_ddl_H
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_H

      - name: Test fail_over_ddl_I
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_I
      
      - name: Test fail_over_ddl_J
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_J

      - name: Test fail_over_ddl_K
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_K

      - name: Test fail_over_ddl_L
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_L
      
      - name: Test fail_over_ddl_M
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_M
          
      - name: Test fail_over_ddl_N
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_N

      - name: Test fail_over_ddl_O
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_O
      
      - name: Test fail_over_ddl_mix
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix 

      - name: Test fail_over_ddl_mix_with_syncpoint
        run: |
          pwd && ls -l bin/ && ls -l tools/bin/
          export TICDC_NEWARCH=true && make integration_test CASE=fail_over_ddl_mix_with_syncpoint 

      - name: Upload test logs
        if: always()
        uses: ./.github/actions/upload-test-logs
        with:
          log-name: failover_e2e_test1