CI/CD checks for integration test #22
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Modified Connectors | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| paths: | |
| - 'airbyte-integrations/connectors/**' | |
| workflow_dispatch: | |
| jobs: | |
| test-connectors: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Ensure full commit history is fetched | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| - name: Add Upstream Remote (Fork Fix) | |
| run: | | |
| git remote add upstream https://github.com/avirajsingh7/airbyte.git | |
| git fetch upstream | |
| git branch --all | |
| - name: Get Start Timestamp | |
| id: get-start-timestamp | |
| shell: bash | |
| run: echo "start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT | |
| - name: Install Java Environment | |
| id: install-java-environment | |
| uses: ./.github/actions/install-java-environment | |
| - name: Install Airbyte CI | |
| id: install-airbyte-ci | |
| uses: ./.github/actions/install-airbyte-ci | |
| with: | |
| airbyte_ci_binary_url: "https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci" | |
| is_fork: true | |
| # Get the base commit SHA from upstream/main (Airbyte Repo) | |
| - name: Get base commit SHA | |
| id: base_sha | |
| run: | | |
| BASE_SHA=$(git merge-base upstream/${{ github.event.pull_request.base.ref }} HEAD) | |
| echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV | |
| echo "Base SHA: $BASE_SHA" | |
| # Get the head commit SHA (source branch) | |
| - name: Get head commit SHA | |
| id: head_sha | |
| run: | | |
| HEAD_SHA=$(git rev-parse HEAD) | |
| echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_ENV | |
| echo "Head SHA: $HEAD_SHA" | |
| - name: Get Modified Connector Folders | |
| id: modified-connectors | |
| run: | | |
| echo "::group::Identifying modified connectors" | |
| echo "Running git diff to get modified files:" | |
| git diff --name-only $BASE_SHA $HEAD_SHA | |
| MODIFIED_FOLDERS=$(git diff --name-only $BASE_SHA $HEAD_SHA | grep '^airbyte-integrations/connectors/' | cut -d '/' -f 3 | sort | uniq) | |
| echo "Modified folders:" | |
| echo "$MODIFIED_FOLDERS" | |
| echo "FOLDERS=$MODIFIED_FOLDERS" >> $GITHUB_ENV | |
| echo "::endgroup::" | |
| - name: Install Setuptools | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools | |
| - name: Process Each Connector | |
| if: env.FOLDERS != '' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| echo "$FOLDERS" | while read -r connector; do | |
| echo "Processing connector: $connector" | |
| # Create the secret config.json path | |
| CONFIG_PATH="airbyte-integrations/connectors/${connector}/secrets" | |
| mkdir -p "$CONFIG_PATH" | |
| # Fetch secrets from GitHub | |
| SECRET_KEY="SECRET$(echo "${connector^^}" | tr -d '-')" | |
| echo "Looking for secret: $SECRET_KEY" | |
| SECRET_VALUE="${{ secrets.SECRETSOURCEAVNI }}" | |
| if [ -z "$SECRET_VALUE" ]; then | |
| echo "No secrets found for $connector. Skipping." | |
| exit 1 | |
| fi | |
| # Write the secret to config.json | |
| echo "$SECRET_VALUE" > "$CONFIG_PATH/config.json" | |
| # Run airbyte-ci test command | |
| echo "Running tests for $connector" | |
| airbyte-ci connectors --name=$connector test | |
| # Clean up the folder | |
| echo "Cleaning up secrets for $connector" | |
| rm -rf "$CONFIG_PATH" | |
| done |