CI/CD checks for integration test #32
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Modified Connectors | ||
| on: | ||
| pull_request: | ||
| types: | ||
| - opened | ||
| - reopened | ||
| - synchronize | ||
| paths: | ||
| - 'airbyte-integrations/connectors/**' | ||
| workflow_dispatch: | ||
| jobs: | ||
| test-connectors: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout Code | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 # Ensure full commit history is fetched | ||
| ref: ${{ github.event.pull_request.head.ref }} | ||
| - name: Add Upstream Remote (Fork Fix) | ||
| run: | | ||
| git remote add upstream https://github.com/avirajsingh7/airbyte.git | ||
| git fetch upstream | ||
| git branch --all | ||
| - name: Get Start Timestamp | ||
| id: get-start-timestamp | ||
| shell: bash | ||
| run: echo "start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT | ||
| # Get the base commit SHA from upstream/main (Airbyte Repo) | ||
| - name: Get base commit SHA | ||
| id: base_sha | ||
| run: | | ||
| BASE_SHA=$(git merge-base upstream/${{ github.event.pull_request.base.ref }} HEAD) | ||
| echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV | ||
| echo "Base SHA: $BASE_SHA" | ||
| # Get the head commit SHA (source branch) | ||
| - name: Get head commit SHA | ||
| id: head_sha | ||
| run: | | ||
| HEAD_SHA=$(git rev-parse HEAD) | ||
| echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_ENV | ||
| echo "Head SHA: $HEAD_SHA" | ||
| - name: Get Modified Connector Folders | ||
| id: modified-connectors | ||
| run: | | ||
| echo "::group::Identifying modified connectors" | ||
| echo "Running git diff to get modified files:" | ||
| git diff --name-only $BASE_SHA $HEAD_SHA | ||
| MODIFIED_FOLDERS=$(git diff --name-only $BASE_SHA $HEAD_SHA | grep '^airbyte-integrations/connectors/' | cut -d '/' -f 3 | sort | uniq) | ||
| echo "Modified folders:" | ||
| echo "$MODIFIED_FOLDERS" | ||
| echo "FOLDERS=$MODIFIED_FOLDERS" >> $GITHUB_ENV | ||
| echo "::endgroup::" | ||
| - name: Process Each Connector | ||
| if: env.FOLDERS != '' | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| run: | | ||
| # Ensure we're in the correct directory | ||
| cd $GITHUB_WORKSPACE | ||
| CONNECTORS_TO_TEST="" | ||
| echo "$FOLDERS" | while read -r connector; do | ||
| echo "Processing connector: $connector" | ||
| # Create the secret config.json path | ||
| CONFIG_PATH="airbyte-integrations/connectors/${connector}/secrets" | ||
| mkdir -p "$CONFIG_PATH" | ||
| # Fetch secrets from GitHub | ||
| SECRET_KEY="SECRET$(echo "${connector^^}" | tr -d '-')" | ||
| echo "Looking for secret: $SECRET_KEY" | ||
| SECRET_VALUE="${{ secrets.SECRETSOURCEAVNI }}" | ||
| if [ -z "$SECRET_VALUE" ]; then | ||
| echo "No secrets found for $connector. Skipping." | ||
| continue | ||
| fi | ||
| # Write the secret to config.json | ||
| echo "$SECRET_VALUE" > "$CONFIG_PATH/config.json" | ||
| # Add connector to the list of connectors to test | ||
| CONNECTORS_TO_TEST="$CONNECTORS_TO_TEST --name=$connector" | ||
| done | ||
| # Set the CONNECTORS_TO_TEST variable for the next step | ||
| echo "CONNECTORS_TO_TEST=$CONNECTORS_TO_TEST" >> $GITHUB_ENV | ||
| - name: Test connectors [PULL REQUESTS] | ||
| if: github.event_name == 'pull_request' | ||
| uses: ./.github/actions/run-airbyte-ci | ||
| with: | ||
| context: "pull_request" | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| git_repo_url: "https://github.com/avirajsingh7/airbyte.git" | ||
| github_token: ${{ secrets.TOKEN }} | ||
| git_branch: ${{ github.head_ref }} | ||
| git_revision: ${{ HEAD_SHA }} | ||
|
Check failure on line 112 in .github/workflows/tech4dev-ci-test.yml
|
||
| subcommand: "connectors ${{ env.CONNECTORS_TO_TEST }} --execute-timeout=18000 test" | ||
