airbyte-ci: retry publish on failure (#42959)

airbytehq · Aug 2, 2024 · 9ef365e · 9ef365e
1 parent 4e6fc42
commit 9ef365e
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 3 deletions.
diff --git a/.github/actions/run-airbyte-ci/action.yml b/.github/actions/run-airbyte-ci/action.yml
@@ -87,6 +87,15 @@ inputs:
     description: "Whether the PR is from a fork"
     required: false
     default: "false"
+  max_attempts:
+    description: "Number of attempts at running the airbyte-ci command"
+    required: false
+    default: 1
+  retry_wait_seconds:
+    description: "Number of seconds to wait between retry attempts"
+    required: false
+    default: 60
+
 runs:
   using: "composite"
   steps:
@@ -109,7 +118,7 @@ runs:
         is_fork: ${{ inputs.is_fork }}
     - name: Run airbyte-ci
       id: run-airbyte-ci
-      shell: bash
+      uses: nick-fields/retry@v3
       env:
         CI: "True"
         CI_GIT_USER: ${{ github.repository_owner }}
@@ -140,8 +149,14 @@ runs:
         SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }}
         SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }}
         SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }}
-      run: |
-        airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+      with:
+        shell: bash
+        max_attempts: ${{ inputs.max_attempts }}
+        retry_wait_seconds: ${{ inputs.retry_wait_seconds }}
+        # 360mn > 6 hours: it's the GitHub runner max job duration
+        timeout_minutes: 360
+        command: |
+          airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
     - name: Stop Engine
       id: stop-engine
       if: always()

diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml
@@ -44,6 +44,7 @@ jobs:
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
           python_registry_token: ${{ secrets.PYPI_TOKEN }}
+          max_attempts: 2
 
       - name: Publish connectors [manual]
         id: publish-connectors
@@ -66,6 +67,7 @@ jobs:
           subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"
           python_registry_token: ${{ secrets.PYPI_TOKEN }}
           airbyte_ci_binary_url: ${{ github.event.inputs.airbyte_ci_binary_url }}
+          max_attempts: 2
 
   set-instatus-incident-on-failure:
     name: Create Instatus Incident on Failure