Merge branch 'master' into feat/upgrade_milvus_destination_cdk_version

.github/actions/install-airbyte-ci/action.yml

@@ -30,21 +30,26 @@ runs:
     - name: "Determine how Airbyte CI should be installed"
       shell: bash
       id: determine-install-mode
-      # When the PR is from a fork, we always install from binary
-      if: inputs.is_fork == 'false'
       run: |
-        if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
-          echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
-          echo "install-mode=source" >> $GITHUB_OUTPUT
-          echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV
-        else
-          echo "install-mode=binary" >> $GITHUB_OUTPUT
-          echo "SENTRY_ENVIRONMENT=production" >> $GITHUB_ENV
-        fi
+        echo "install-mode=source" >> $GITHUB_OUTPUT
+        echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV
+
+      # When the PR is from a fork, we always install from binary
+      # if: inputs.is_fork == 'false'
+      # run: |
+      #   if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
+      #     echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
+      #     echo "install-mode=source" >> $GITHUB_OUTPUT
+      #     echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV
+      #   else
+      #     echo "install-mode=binary" >> $GITHUB_OUTPUT
+      #     echo "SENTRY_ENVIRONMENT=production" >> $GITHUB_ENV
+      #   fi
 
     - name: Install Airbyte CI from binary
       id: install-airbyte-ci-binary
-      if: steps.determine-install-mode.outputs.install-mode == 'binary' || ${{ inputs.is_fork }} == 'true'
+      if: false
+      # if: steps.determine-install-mode.outputs.install-mode == 'binary' || ${{ inputs.is_fork }} == 'true'
       shell: bash
       run: |
         curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
@@ -54,21 +59,27 @@ runs:
     - name: Install Python 3.10
       id: install-python-3-10
       uses: actions/setup-python@v4
-      if: steps.determine-install-mode.outputs.install-mode == 'source'
+      # if: steps.determine-install-mode.outputs.install-mode == 'source'
       with:
         python-version: "3.10"
         token: ${{ inputs.github_token }}
 
     - name: Install Airbyte CI from source
       id: install-airbyte-ci-source
-      if: steps.determine-install-mode.outputs.install-mode == 'source'
+      if: true
+      # if: steps.determine-install-mode.outputs.install-mode == 'source'
       shell: bash
       run: |
         pip install --upgrade pip
         pip install pipx
         pipx ensurepath
         pipx install ${{ inputs.path_to_airbyte_ci_source }}
 
+    - name: Print installed `airbyte-ci` version
+      shell: bash
+      run: |
+        airbyte-ci --version
+
     - name: Get dagger engine image name
       id: get-dagger-engine-image-name
       shell: bash

.github/actions/run-airbyte-ci/action.yml

@@ -195,11 +195,17 @@ runs:
       shell: bash
       run: tar cvzf ./dagger_engine_logs.tgz ./dagger_engine_logs
 
+    - name: Hash subcommand
+      id: hash-subcommand
+      shell: bash
+      if: always()
+      run: echo "subcommand_hash=$(echo ${{ inputs.subcommand }} | sha256sum | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
+
     - name: Upload logs to GitHub
       id: upload-dagger-engine-logs
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ github.job }}_dagger_engine_logs.tgz
+        name: ${{ github.job }}_${{ steps.hash-subcommand.outputs.subcommand_hash }}_dagger_engine_logs.tgz
         path: ./dagger_engine_logs.tgz
         retention-days: 7

.github/workflows/airbyte-ci-release.yml

@@ -43,6 +43,8 @@ jobs:
       - name: Install Poetry
         id: install_poetry
         uses: snok/install-poetry@v1
+        with:
+          version: 1.8.5
 
       - name: Install Dependencies
         id: install_dependencies
@@ -54,7 +56,7 @@ jobs:
         working-directory: airbyte-ci/connectors/pipelines/
         run: poetry run poe build-release-binary ${{ env.BINARY_FILE_NAME }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: airbyte-ci-${{ matrix.os }}-${{ steps.get_short_sha.outputs.sha }}
           path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
@@ -87,7 +89,7 @@ jobs:
         if: github.ref == 'refs/heads/master'
         working-directory: airbyte-ci/connectors/pipelines/
         run: |
-          echo "::set-output name=version::$(poetry version --short)"
+          echo "version=$(poetry version --short)" >> $GITHUB_OUTPUT
 
       - name: Authenticate to Google Cloud Prod
         id: auth_prod

.github/workflows/airbyte-ci-tests.yml

@@ -17,7 +17,7 @@ on:
       - synchronize
 jobs:
   changes:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     outputs:
       internal_poetry_packages: ${{ steps.changes.outputs.internal_poetry_packages }}
 

.github/workflows/approve-and-merge-dispatch.yml

@@ -4,7 +4,7 @@ on:
     types: [created]
 jobs:
   approveAndMergeDispatch:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Auto Approve Slash Command Dispatch
         uses: peter-evans/slash-command-dispatch@v3

.github/workflows/approve-regression-tests-command.yml

@@ -31,7 +31,7 @@ run-name: "Approve Regression Tests #${{ github.event.inputs.pr }}"
 jobs:
   approve-regression-tests:
     name: "Approve Regression Tests"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Get job variables
         id: job-vars

.github/workflows/assign-issue-to-project.yaml

@@ -9,7 +9,7 @@ env:
 
 jobs:
   assign_one_project:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     name: Assign to Airbyte Github Project
     steps:
       - name: Assign documentation issues to the Documentation Roadmap project

.github/workflows/auto_merge.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   run_auto_merge:
     name: Run auto-merge
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -18,6 +18,8 @@ jobs:
           python-version: "3.10"
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
+        with:
+          version: 1.8.5
       - name: Run auto merge
         shell: bash
         working-directory: airbyte-ci/connectors/auto_merge

.github/workflows/bump-version-command.yml

@@ -41,7 +41,7 @@ concurrency:
 jobs:
   bump-version:
     name: "Bump version of connectors in this PR"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Get job variables
         id: job-vars
@@ -100,7 +100,7 @@ jobs:
       - name: Check for changes
         id: git-diff
         run: |
-          git diff --quiet && echo "No changes to commit" || echo "::set-output name=changes::true"
+          git diff --quiet && echo "No changes to commit" || echo "changes=true" >> $GITHUB_OUTPUT
         shell: bash
 
       # Commit changes (if any)

.github/workflows/community_ci.yml

@@ -19,7 +19,7 @@ jobs:
   fail_on_protected_path_changes:
     name: "Check fork do not change protected paths"
     if: github.event.pull_request.head.repo.fork == true
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       pull-requests: read
     steps:
@@ -37,45 +37,6 @@ jobs:
           echo "The fork has changes in protected paths. This is not allowed."
           exit 1
 
-  format_check:
-    # IMPORTANT: This name must match the require check name on the branch protection settings
-    name: "Check for formatting errors"
-    if: github.event.pull_request.head.repo.fork == true
-    environment: community-ci-auto
-    runs-on: community-tooling-test-small
-    needs: fail_on_protected_path_changes
-    timeout-minutes: 30
-    env:
-      MAIN_BRANCH_NAME: "master"
-    steps:
-      # This checkouts a fork which can contain untrusted code
-      # It's deemed safe as the formatter are not executing any checked out code
-      - name: Checkout fork
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 1
-
-      # This will sync the .github folder of the main repo with the fork
-      # This allows us to use up to date actions and CI logic from the main repo
-      - name: Pull .github folder and internal packages from main repository
-        id: pull_github_folder
-        run: |
-          git remote add main https://github.com/airbytehq/airbyte.git
-          git fetch main ${MAIN_BRANCH_NAME}
-          git checkout main/${MAIN_BRANCH_NAME} -- .github
-          git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
-
-      - name: Run airbyte-ci format check all
-        # This path refers to the fork .github folder.
-        # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          context: "pull_request"
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          subcommand: "format check all"
-          is_fork: "true"
   connectors_early_ci:
     name: Run connectors early CI on fork
     if: github.event.pull_request.head.repo.fork == true

.github/workflows/connector-performance-command.yml

@@ -92,15 +92,15 @@ jobs:
   uuid:
     name: "Custom UUID of workflow run"
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: UUID ${{ inputs.uuid }}
         run: true
   start-test-runner:
     name: Start Build EC2 Runner
     needs: uuid
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
@@ -262,7 +262,7 @@ jobs:
       - start-test-runner # required to get output from the start-runner job
       - performance-test # required to wait when the main job is done
       - uuid
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
     steps:
       - name: Configure AWS credentials

.github/workflows/connector_code_freeze.yml

@@ -18,7 +18,7 @@ env:
   CODE_FREEZE_END_DATE: "2024-01-02"
 jobs:
   code-freeze-check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     name: Code freeze check
     permissions:
       # This is required to be able to comment on PRs and list changed files
@@ -35,10 +35,10 @@ jobs:
 
           if [ "$current_date" -ge "$start_date" ] && [ "$current_date" -le "$end_date" ]; then
               echo "Code freeze is in effect"
-              echo "::set-output name=is_in_code_freeze::true"
+              echo "is_in_code_freeze=true" >> $GITHUB_OUTPUT
           else
               echo "Code freeze is not in effect"
-              echo "::set-output name=is_in_code_freeze::false"
+              echo "is_in_code_freeze=false" >> $GITHUB_OUTPUT
           fi
 
       # Use GitHub PR Api to get the list of changed files

.github/workflows/connector_teams_review_requirements.yml

@@ -17,7 +17,7 @@ on:
 jobs:
   check-review-requirements:
     name: "Check if a review is required from Connector teams"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     if: ${{ github.event.pull_request.head.repo.fork == false && github.event.pull_request.draft == false }}
     steps:

.github/workflows/connectors_insights.yml

@@ -31,6 +31,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: 1.8.5
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

.github/workflows/connectors_nightly_build.yml

@@ -5,18 +5,38 @@ on:
     # 0AM UTC is 2AM CEST, 3AM EEST, 5PM PDT.
     - cron: "0 0 * * *"
   workflow_dispatch:
-    inputs:
-      test-connectors-options:
-        default: --concurrency=5 --support-level=certified
-        required: true
 
 run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}"
 
 jobs:
+  generate_matrix:
+    name: Generate matrix
+    runs-on: ubuntu-24.04
+    outputs:
+      generated_matrix: ${{ steps.generate_matrix.outputs.generated_matrix }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v4
+      - name: Run airbyte-ci connectors list [SCHEDULED TRIGGER]
+        id: airbyte-ci-connectors-list-scheduled
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          context: "master"
+          subcommand: "connectors --support-level=certified list --output=selected_connectors.json"
+      - name: Generate matrix - 30 connectors per job
+        id: generate_matrix
+        run: |
+          matrix=$(jq -c -r '{include: [.[] | "--name=" + .] | to_entries | group_by(.key / 30 | floor) | map(map(.value) | {"connector_names": join(" ")})}' selected_connectors.json)
+          echo "generated_matrix=$matrix" >> $GITHUB_OUTPUT
+
   test_connectors:
+    needs: generate_matrix
     name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}"
-    timeout-minutes: 720 # 12 hours
     runs-on: connector-nightly-xlarge
+    continue-on-error: true
+    strategy:
+      matrix: ${{fromJson(needs.generate_matrix.outputs.generated_matrix)}}
+
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -32,7 +52,7 @@ jobs:
         with:
           context: "master"
           ci_job_key: "nightly_builds"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }}
+          # dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -41,4 +61,4 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=8 --support-level=certified' }} test"
+          subcommand: "connectors ${{ matrix.connector_names}} test"

.github/workflows/connectors_tests.yml

@@ -23,7 +23,7 @@ on:
       - synchronize
 jobs:
   changes:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     outputs:
       connectors: ${{ steps.changes.outputs.connectors }}
     permissions:
@@ -78,7 +78,7 @@ jobs:
     # If the condition is not met the job will be skipped (it will not fail)
     if: (github.event_name == 'pull_request' && needs.changes.outputs.connectors == 'true' && github.event.pull_request.head.repo.fork != true) || github.event_name == 'workflow_dispatch'
     name: Connectors CI
-    runs-on: connector-test-large
+    runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
     timeout-minutes: 360 # 6 hours
     steps:
       - name: Checkout Airbyte