10_integrate_airflow_changes.yml

# orchestrate-workflow.yml
name: 🎯 Airflow Validations

on:  # yamllint disable-line rule:truthy
  pull_request:
    paths:
      - orchestrate/*
      - orchestrate/**/*

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# This cancels a run if another change is pushed to the same branch
concurrency:
  group: orchestrate-${{ github.ref }}
  cancel-in-progress: true

jobs:
  airflow:
    name: Pull Request Airflow Tests
    runs-on: ubuntu-latest
    # container: datacoves/ci-airflow-dbt-snowflake:3.3
    container: datacoves/ci-airflow-dbt-snowflake:3.3

    env:
      AIRFLOW__CORE__DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/orchestrate/dags
      AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 300
      AIRFLOW__ARTIFACTS_PATH: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/orchestrate
      DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt
      DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform
      DATACOVES__REPO_PATH: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
      PYTHONPATH: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
      FORCE_COLOR: 1
      OUTPUT_FILE: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/test_output.md

    steps:
      - name: Checkout branch
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Test DAGs Load time and variable usage at top level
        id: test_load_time_variables
        run: python /usr/app/test_dags.py --dag-loadtime-threshold 1 --check-variable-usage --write-output --filename "$OUTPUT_FILE"

      # if write-output is set in the prior step, the following step will run
      - name: Add PR comment of results of test_load_time_variables tests
        uses: thollander/actions-comment-pull-request@v2
        with:
          filePath: ${{ env.OUTPUT_FILE }}
          comment_tag: Test DAGs Load time and variable usage at top level

      - name: Custom Airflow Validation Tests
        env:
          NO_COLOR: 1
        run: pytest $AIRFLOW__ARTIFACTS_PATH/test_dags/validate_dags.py --output-file "$OUTPUT_FILE"


      - name: Add PR comment of results of custom Airflow validation tests
        if: always()
        uses: thollander/actions-comment-pull-request@v2
        with:
          # filePath: formatted_output.md
          filePath: ${{ env.OUTPUT_FILE }}
          comment_tag: Custom Tests
          GITHUB_TOKEN: ${{ github.token }}