From 8be1022b2cf4c76fbf1589578101b85b1314e6c4 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Fri, 11 Aug 2023 12:12:58 +0100 Subject: [PATCH 1/6] Improve CI tests runtime --- .github/workflows/test.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68bbaa3d7..6750fdf3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,12 @@ jobs: airflow-version: ["2.3", "2.4", "2.5", "2.6"] steps: - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .nox + key: unit-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -62,7 +68,12 @@ jobs: - 5432:5432 steps: - uses: actions/checkout@v3 - + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .nox + key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -112,7 +123,12 @@ jobs: steps: - uses: actions/checkout@v3 if: github.event_name != 'pull_request_target' - + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .nox + key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Checkout pull/${{ github.event.number }} uses: actions/checkout@v3 with: From 83d9e335243df3bce919f20bcde292cc2f0e1615 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Fri, 11 Aug 2023 12:44:20 +0100 Subject: [PATCH 2/6] Move virtualenv test to expensive integration tests suite --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c1c68f36f..5b02c7e9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,7 +162,7 @@ test-integration = """pytest -vv \ --cov-report=xml \ --durations=0 \ -m integration \ --k 'not test_example_dag[example_cosmos_python_models]' +-k 'not (test_example_dag[example_cosmos_python_models] or test_example_dag[example_virtualenv])' """ test-integration-expensive = """pytest -vv \ --cov=cosmos \ @@ -170,7 +170,7 @@ test-integration-expensive = """pytest -vv \ --cov-report=xml \ --durations=0 \ -m integration \ --k 'test_example_dag[example_cosmos_python_models]'""" +-k 'test_example_dag[example_cosmos_python_models] or test_example_dag[example_virtualenv])'""" [tool.pytest.ini_options] filterwarnings = [ From 7f1edb50f3fbbfbf1fb5c0023b584f21a54a8701 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Fri, 11 Aug 2023 13:00:55 +0100 Subject: [PATCH 3/6] Fix issue with integration tests selector --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5b02c7e9e..72ae53c61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,7 +170,7 @@ test-integration-expensive = """pytest -vv \ --cov-report=xml \ --durations=0 \ -m integration \ --k 'test_example_dag[example_cosmos_python_models] or test_example_dag[example_virtualenv])'""" +-k 'test_example_dag[example_cosmos_python_models] or test_example_dag[example_virtualenv]'""" [tool.pytest.ini_options] filterwarnings = [ From 8c4532eec28d1cebf36234cf97ab9d0302017165 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 14 Aug 2023 11:58:08 +0100 Subject: [PATCH 4/6] Fix issue with expensive integration tests setup --- .github/workflows/test.yml | 43 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6750fdf3a..c4c8c5a01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,11 +54,9 @@ jobs: airflow-version: ["2.3", "2.4", "2.5", "2.6"] services: postgres: - # Docker Hub image image: postgres env: POSTGRES_PASSWORD: postgres - # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready --health-interval 10s @@ -110,6 +108,18 @@ jobs: matrix: python-version: ["3.10"] airflow-version: ["2.6"] + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 if: >- github.event_name == 'push' || ( @@ -164,3 +174,32 @@ jobs: DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + + Code-Coverage: + if: github.event.action != 'labeled' + needs: + - Run-Unit-Tests + - Run-Integration-Tests + - Run-Integration-Tests-Expensive + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install coverage + run: | + pip3 install coverage + - name: Download all artifacts + uses: actions/download-artifact@v2 + - name: Run coverage + run: | + coverage combine ./coverage/coverage*/.coverage + coverage report + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} From c38b10c6946e959b6941ef5ea9068b3af7ce11a7 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 14 Aug 2023 12:32:52 +0100 Subject: [PATCH 5/6] Remove codecov change --- .github/workflows/test.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c4c8c5a01..fa23bb333 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -174,32 +174,3 @@ jobs: DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} - - Code-Coverage: - if: github.event.action != 'labeled' - needs: - - Run-Unit-Tests - - Run-Integration-Tests - - Run-Integration-Tests-Expensive - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - name: Install coverage - run: | - pip3 install coverage - - name: Download all artifacts - uses: actions/download-artifact@v2 - - name: Run coverage - run: | - coverage combine ./coverage/coverage*/.coverage - coverage report - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} From 3e58da4c77fd91bb15b02033d186828de913c681 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 15:26:18 +0000 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=8E=A8=20[pre-commit.ci]=20Auto=20for?= =?UTF-8?q?mat=20from=20pre-commit.com=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3e543b084..0f5f9d56c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -89,7 +89,7 @@ jobs: ~/.cache/pip .nox key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: