From 1b13cf50b6487c833be71e23c696b9aa89824941 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 29 May 2024 14:31:29 +0200 Subject: [PATCH] Another attempt to stabilize (and limit) integration tests (#39913) There is no particular reason we should run the integration tests for both Postgres and mysql - and Postgres ones seem to be much more stable in general, so we only limit the tests to be run the tests for Postgres. Also - since the tests (especially mssql) seem to be flaky, we re-run them once if they fail --- .github/workflows/integration-tests.yml | 21 +++----- .../run_integration_tests_with_retry.sh | 53 +++++++++++++++++++ .../cli/commands/test_celery_command.py | 2 +- .../executors/test_celery_executor.py | 4 +- 4 files changed, 64 insertions(+), 16 deletions(-) create mode 100755 scripts/ci/testing/run_integration_tests_with_retry.sh diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8dd4cae6a2928..df9b06fbfc31a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -59,20 +59,18 @@ on: # yamllint disable-line rule:truthy jobs: tests-integration: timeout-minutes: 130 - name: Integration Tests ${{ matrix.backend }}-${{ matrix.integration }} + name: "Integration Tests: ${{ matrix.integration }}" runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} strategy: fail-fast: false matrix: - backend: ["postgres", "mysql"] integration: ${{ fromJSON(inputs.testable-integrations) }} env: IMAGE_TAG: "${{ inputs.image-tag }}" - BACKEND: "${{ matrix.backend }}" - # yamllint disable-line rule:line-length - BACKEND_VERSION: ${{ matrix.backend == 'postgres' && inputs.default-postgres-version || inputs.default-mysql-version }}" + BACKEND: "postgres" + BACKEND_VERSION: ${{ inputs.default-postgres-version }}" PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - JOB_ID: "integration-${{ matrix.backend }}-${{ matrix.integration }}" + JOB_ID: "integration-${{ matrix.integration }}" SKIP_PROVIDER_TESTS: "${{ inputs.skip-provider-tests }}" ENABLE_COVERAGE: "${{ inputs.run-coverage}}" DEBUG_RESOURCES: "${{ inputs.debug-resources }}" @@ -92,13 +90,10 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}" uses: ./.github/actions/prepare_breeze_and_image - - name: "Integration Tests ${{ matrix.backend }}:${{ matrix.integration }}" - run: | - breeze down - breeze testing integration-tests --integration ${{ matrix.integration }} - breeze down - - name: "Post Tests success: Integration Tests ${{ matrix.backend }}-${{ matrix.integration }}" + - name: "Integration Tests: ${{ matrix.integration }}" + run: ./scripts/ci/testing/run_integration_tests_with_retry.sh ${{ matrix.integration }} + - name: "Post Tests success: Integration Tests ${{ matrix.integration }}" uses: ./.github/actions/post_tests_success - - name: "Post Tests failure: Integration Tests ${{ matrix.backend }}-${{ matrix.integration }}" + - name: "Post Tests failure: Integration Tests ${{ matrix.integration }}" uses: ./.github/actions/post_tests_failure if: failure() diff --git a/scripts/ci/testing/run_integration_tests_with_retry.sh b/scripts/ci/testing/run_integration_tests_with_retry.sh new file mode 100755 index 0000000000000..4fd25a75ecdff --- /dev/null +++ b/scripts/ci/testing/run_integration_tests_with_retry.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export COLOR_RED=$'\e[31m' +export COLOR_YELLOW=$'\e[33m' +export COLOR_RESET=$'\e[0m' + +if [[ ! "$#" -eq 1 ]]; then + echo "${COLOR_RED}You must provide exactly one argument!.${COLOR_RESET}" + exit 1 +fi + +INTEGRATION=${1} + +breeze down +set +e +breeze testing integration-tests --integration "${INTEGRATION}" +RESULT=$? +set -e +if [[ ${RESULT} != "0" ]]; then + echo + echo "${COLOR_YELLOW}Integration Tests failed. Retrying once${COLOR_RESET}" + echo + echo "This could be due to a flaky test, re-running once to re-check it After restarting docker." + echo + sudo service docker restart + breeze down + set +e + breeze testing integration-tests --integration "${INTEGRATION}" + RESULT=$? + set -e + if [[ ${RESULT} != "0" ]]; then + echo + echo "${COLOR_RED}The integration tests failed for the second time! Giving up${COLOR_RESET}" + echo + exit ${RESULT} + fi +fi diff --git a/tests/integration/cli/commands/test_celery_command.py b/tests/integration/cli/commands/test_celery_command.py index 9ae90ac4d72a3..61866184337d0 100644 --- a/tests/integration/cli/commands/test_celery_command.py +++ b/tests/integration/cli/commands/test_celery_command.py @@ -29,7 +29,7 @@ @pytest.mark.integration("celery") -@pytest.mark.backend("mysql", "postgres") +@pytest.mark.backend("postgres") class TestWorkerServeLogs: @classmethod def setup_class(cls): diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index 2b72d2e91d1a2..64c0a4adc428a 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -97,7 +97,7 @@ def _prepare_app(broker_url=None, execute=None): @pytest.mark.integration("celery") -@pytest.mark.backend("mysql", "postgres") +@pytest.mark.backend("postgres") class TestCeleryExecutor: def setup_method(self) -> None: db.clear_db_runs() @@ -276,7 +276,7 @@ def __ne__(self, other): @pytest.mark.integration("celery") -@pytest.mark.backend("mysql", "postgres") +@pytest.mark.backend("postgres") class TestBulkStateFetcher: bulk_state_fetcher_logger = "airflow.providers.celery.executors.celery_executor_utils.BulkStateFetcher"