diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index e00cdc5fd0d0b..72d4b31418add 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -36,7 +36,7 @@ env: # Airflow one is going to be used CONSTRAINTS_GITHUB_REPOSITORY: >- ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && - secrets.CONSTRAINTS_GITHUB_REPOSITORY || github.repository }} + secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} # This token is WRITE one - workflow_run type of events always have the WRITE token GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token should not be empty in workflow_run type of event. @@ -409,8 +409,8 @@ jobs: DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }} FORCE_PULL_BASE_PYTHON_IMAGE: > ${{ needs.cancel-workflow-runs.sourceEvent == 'schedule' && 'true' || 'false' }} - VERSION_SUFFIX_FOR_PYPI: "dev" - VERSION_SUFFIX_FOR_SVN: "dev" + VERSION_SUFFIX_FOR_PYPI: ".dev0" + VERSION_SUFFIX_FOR_SVN: ".dev0" steps: - name: > Checkout [${{ needs.cancel-workflow-runs.outputs.sourceEvent }}] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a713194947af9..927a403097c41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ env: # Airflow one is going to be used CONSTRAINTS_GITHUB_REPOSITORY: >- ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && - secrets.CONSTRAINTS_GITHUB_REPOSITORY || github.repository }} + secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # In builds from forks, this token is empty, and this is good because such builds do not even try @@ -520,8 +520,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} AIRFLOW_EXTRAS: "all" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - VERSION_SUFFIX_FOR_PYPI: "dev" - VERSION_SUFFIX_FOR_SVN: "dev" + VERSION_SUFFIX_FOR_PYPI: ".dev0" + VERSION_SUFFIX_FOR_SVN: ".dev0" GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'master' steps: @@ -545,7 +545,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" env: PACKAGE_FORMAT: "wheel" - name: "Prepare airflow package: wheel" - run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh + run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh env: PACKAGE_FORMAT: "wheel" - name: "Install and test provider packages and airflow via wheel files" @@ -553,11 +553,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" env: USE_AIRFLOW_VERSION: "wheel" PACKAGE_FORMAT: "wheel" - - name: "Install and test provider packages and airflow on Airflow 2.0 files" + - name: "Install and test provider packages and airflow on Airflow 2.1 files" run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh env: - USE_AIRFLOW_VERSION: "2.0.0" + USE_AIRFLOW_VERSION: "2.1.0" PACKAGE_FORMAT: "wheel" + if: false prepare-test-provider-packages-sdist: timeout-minutes: 40 @@ -568,8 +569,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} AIRFLOW_EXTRAS: "all" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - VERSION_SUFFIX_FOR_PYPI: "dev" - VERSION_SUFFIX_FOR_SVN: "dev" + VERSION_SUFFIX_FOR_PYPI: ".dev0" + VERSION_SUFFIX_FOR_SVN: ".dev0" GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'master' steps: @@ -591,7 +592,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" env: PACKAGE_FORMAT: "sdist" - name: "Prepare airflow package: sdist" - run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh + run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh env: PACKAGE_FORMAT: "sdist" - name: "Install and test provider packages and airflow via sdist files" diff --git a/.markdownlint.yml b/.markdownlint.yml index b30e3d38ae7cb..69793feedb1f1 100644 --- a/.markdownlint.yml +++ b/.markdownlint.yml @@ -16,41 +16,42 @@ # under the License. # --- -# MD004/ul-style +# https://github.com/DavidAnson/markdownlint#rules--aliases +# MD004 ul-style - Unordered list style MD004: false -# MD007/ul-indent +# MD007 ul-indent - Unordered list indentation MD007: false -# MD012/no-multiple-blanks +# MD012 no-multiple-blanks - Multiple consecutive blank lines MD012: false -# MD013 Line length +# MD013 line-length - Line length MD013: false -# MD024/no-duplicate-heading/no-duplicate-header +# MD024 no-duplicate-heading/no-duplicate-header - Multiple headings with the same content MD024: false -# MD026/no-trailing-punctuation +# MD026 no-trailing-punctuation - Trailing punctuation in heading MD026: false -# MD029/ol-prefix +# MD029 ol-prefix - Ordered list item prefix MD029: false -# MD030/list-marker-space +# MD030 list-marker-space - Spaces after list markers MD030: false -# MD033/no-inline-html +# MD033 no-inline-html - Inline HTML MD033: false -# MD034/no-bare-urls +# MD034 no-bare-urls - Bare URL used MD034: false -# MD036/no-emphasis-as-heading/no-emphasis-as-header +# MD036 no-emphasis-as-heading/no-emphasis-as-header - Emphasis used instead of a heading MD036: false -# MD040/fenced-code-language +# MD040 fenced-code-language - Fenced code blocks should have a language specified MD040: false -# MD041/first-line-heading/first-line-h1 +# MD041 first-line-heading/first-line-h1 - First line in a file should be a top-level heading MD041: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b68511b7d43b2..5abcaa12a00b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -92,7 +92,7 @@ repos: - license-templates/LICENSE.txt - --fuzzy-match-generates-todo - id: insert-license - name: Add license for all python files + name: Add license for all Python files exclude: ^\.github/.*$ types: [python] args: @@ -112,7 +112,7 @@ repos: - license-templates/LICENSE.txt - --fuzzy-match-generates-todo - id: insert-license - name: Add license for all yaml files + name: Add license for all YAML files exclude: ^\.github/.*$ types: [yaml] files: \.yml$|\.yaml$ @@ -146,7 +146,7 @@ repos: hooks: - id: doctoc name: Add TOC for md files - files: ^README\.md$|^CONTRIBUTING\.md$|^UPDATING.*.md$|^dev/.*.md$|^dev/.*.rst + files: ^README\.md$|^CONTRIBUTING\.md$|^UPDATING.*\.md$|^dev/.*\.md$|^dev/.*\.rst$ args: - "--maxlevel" - "2" @@ -184,16 +184,16 @@ repos: rev: v1.26.1 hooks: - id: yamllint - name: Check yaml files with yamllint + name: Check YAML files with yamllint entry: yamllint -c yamllint-config.yml --strict types: [yaml] exclude: - ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/(?:templates|files)/.*\.yaml + ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/(?:templates|files)/.*\.yaml$ - repo: https://github.com/timothycrosley/isort rev: 5.8.0 hooks: - id: isort - name: Run isort to sort imports + name: Run isort to sort imports in Python files files: \.py$ # To keep consistent with the global isort skip config defined in setup.cfg exclude: ^build/.*$|^.tox/.*$|^venv/.*$ @@ -230,14 +230,14 @@ repos: - id: lint-dockerfile name: Lint dockerfile language: system - entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh" + entry: ./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh files: Dockerfile.*$ pass_filenames: true require_serial: true - id: setup-order - name: Check order of dependencies in setup.py and setup.cfg + name: Check order of dependencies in setup.cfg and setup.py language: python - files: ^setup.py$|^setup.cfg$ + files: ^setup\.cfg$|^setup\.py$ pass_filenames: false entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py additional_dependencies: ['rich'] @@ -245,31 +245,31 @@ repos: name: Checks setup extra packages description: Checks if all the libraries in setup.py are listed in extra-packages-ref.rst file language: python - files: ^setup.py$|^docs/apache-airflow/extra-packages-ref.rst$ + files: ^setup\.py$|^docs/apache-airflow/extra-packages-ref\.rst$ pass_filenames: false entry: ./scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py additional_dependencies: ['rich==9.2.0'] - id: update-breeze-file name: Update output of breeze command in BREEZE.rst - entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh" + entry: ./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh language: system - files: ^BREEZE.rst$|^breeze$|^breeze-complete$ + files: ^BREEZE\.rst$|^breeze$|^breeze-complete$ pass_filenames: false - id: update-local-yml-file name: Update mounts in the local yml file - entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh" + entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh language: system - files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml" + files: ^scripts/ci/libraries/_local_mounts\.sh$|^scripts/ci/docker_compose/local\.yml$ pass_filenames: false - id: update-setup-cfg-file name: Update setup.cfg file with all licenses - entry: "./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh" + entry: ./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh language: system - files: ^setup.cfg$ + files: ^setup\.cfg$ pass_filenames: false - id: build-providers-dependencies name: Build cross-dependencies for providers packages - entry: "./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh" + entry: ./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh language: system files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$ pass_filenames: false @@ -277,7 +277,7 @@ repos: name: Update extras in documentation entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py language: python - files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$ + files: ^setup\.py$|^INSTALL$|^CONTRIBUTING\.rst$ pass_filenames: false - id: pydevd language: pygrep @@ -315,16 +315,16 @@ repos: pass_filenames: true exclude: > (?x) - ^airflow/providers/apache/cassandra/hooks/cassandra.py$| - ^airflow/providers/apache/hive/operators/hive_stats.py$| + ^airflow/providers/apache/cassandra/hooks/cassandra\.py$| + ^airflow/providers/apache/hive/operators/hive_stats\.py$| ^airflow/providers/apache/hive/.*PROVIDER_CHANGES_*| - ^airflow/providers/apache/hive/.*README.md$| - ^tests/providers/apache/cassandra/hooks/test_cassandra.py$| - ^docs/apache-airflow-providers-apache-cassandra/connections/cassandra.rst$| - ^docs/apache-airflow-providers-apache-hive/commits.rst$| + ^airflow/providers/apache/hive/.*README\.md$| + ^tests/providers/apache/cassandra/hooks/test_cassandra\.py$| + ^docs/apache-airflow-providers-apache-cassandra/connections/cassandra\.rst$| + ^docs/apache-airflow-providers-apache-hive/commits\.rst$| git| ^pylintrc | - ^CHANGELOG.txt$ + ^CHANGELOG\.txt$ - id: consistent-pylint language: pygrep name: Check for inconsistent pylint disable/enable without space @@ -384,17 +384,17 @@ repos: language: pygrep name: "'start_date' should not be defined in default_args in example_dags" entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):" - files: \.*example_dags.*.py$ + files: \.*example_dags.*\.py$ pass_filenames: true - id: check-integrations name: Check if integration list is aligned entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh language: system pass_filenames: false - files: ^common/_common_values.sh$|^breeze-complete$ + files: ^common/_common_values\.sh$|^breeze-complete$ - id: check-apache-license name: Check if licenses are OK for Apache - entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh" + entry: ./scripts/ci/pre_commit/pre_commit_check_license.sh language: system files: ^.*LICENSE.*$|^.*LICENCE.*$ pass_filenames: false @@ -402,7 +402,7 @@ repos: name: Checks for consistency between config.yml and default_config.cfg language: python entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py - files: "config.yml$|default_airflow.cfg$|default.cfg$" + files: config\.yml$|default_airflow\.cfg$|default\.cfg$ pass_filenames: false require_serial: true additional_dependencies: ['pyyaml'] @@ -410,13 +410,13 @@ repos: name: Sort INTHEWILD.md alphabetically entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.sh language: system - files: ^.pre-commit-config.yaml$|^INTHEWILD.md$ + files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$ require_serial: true - id: sort-spelling-wordlist name: Sort alphabetically and uniquify spelling_wordlist.txt entry: ./scripts/ci/pre_commit/pre_commit_sort_spelling_wordlist.sh language: system - files: ^.pre-commit-config.yaml$|^docs/spelling_wordlist.txt$ + files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$ require_serial: true - id: helm-lint name: Lint Helm Chart @@ -433,13 +433,13 @@ repos: - id: bats-tests name: Run BATS bash tests for changed Breeze bash files language: system - entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh tests/bats/breeze/" + entry: ./scripts/ci/pre_commit/pre_commit_bat_tests.sh tests/bats/breeze/ files: ^breeze$|^breeze-complete$|^tests/bats/breeze pass_filenames: false - id: bats-tests name: Run BATS bash tests for changed bash files language: system - entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh" + entry: ./scripts/ci/pre_commit/pre_commit_bat_tests.sh files: \.sh$|\.bash$|\.bats$ exclude: ^tests/bats/in_container|^scripts/in_container|^tests/bats/breeze pass_filenames: true @@ -463,7 +463,7 @@ repos: entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py language: python require_serial: true - files: provider.yaml$|scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py$|^docs/ + files: provider\.yaml$|scripts/ci/pre_commit/pre_commit_check_provider_yaml_files\.py$|^docs/ additional_dependencies: ['PyYAML==5.3.1', 'jsonschema==3.2.0', 'tabulate==0.8.7'] - id: mermaid name: Generate mermaid images @@ -475,7 +475,7 @@ repos: name: Check if pre-commits are described entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh language: system - files: ^.pre-commit-config.yaml$|^STATIC_CODE_CHECKS.rst|^breeze-complete$ + files: ^\.pre-commit-config\.yaml$|^STATIC_CODE_CHECKS\.rst|^breeze-complete$ require_serial: true - id: pre-commit-hook-names name: Ensure hook ids are not overly long @@ -483,7 +483,7 @@ repos: args: - --max-length=70 language: python - files: ^.pre-commit-config.yaml$ + files: ^\.pre-commit-config\.yaml$ additional_dependencies: ['pyyaml'] require_serial: true pass_filenames: false @@ -491,13 +491,13 @@ repos: name: Checks providers available when declared by extras in setup.py language: python entry: ./scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py - files: "setup.py|^airflow/providers/.*.py" + files: setup\.py|^airflow/providers/.*\.py pass_filenames: false require_serial: true additional_dependencies: ['rich'] - id: markdownlint name: Run markdownlint - description: "Checks the style of Markdown files." + description: Checks the style of Markdown files. entry: markdownlint language: node types: [markdown] @@ -533,7 +533,7 @@ repos: - https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json language: python pass_filenames: true - files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose.ya?ml$ + files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose\.ya?ml$ require_serial: true additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0'] - id: json-schema @@ -544,7 +544,7 @@ repos: - chart/values.schema.json language: python pass_filenames: true - files: chart/values.yaml$ + files: chart/values\.yaml$ require_serial: true additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0'] - id: json-schema @@ -555,7 +555,7 @@ repos: - airflow/config_templates/config.yml.schema.json language: python pass_filenames: true - files: airflow/config_templates/config.yml$ + files: airflow/config_templates/config\.yml$ require_serial: true additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0'] - id: flynt @@ -576,26 +576,26 @@ repos: - id: mypy name: Run mypy language: system - entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh" + entry: ./scripts/ci/pre_commit/pre_commit_mypy.sh files: \.py$ exclude: ^dev|^provider_packages|^chart|^docs - id: mypy name: Run mypy for helm chart tests language: system - entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh" + entry: ./scripts/ci/pre_commit/pre_commit_mypy.sh files: ^chart/.*\.py$ require_serial: false - id: mypy name: Run mypy for /docs/ folder language: system - entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh" + entry: ./scripts/ci/pre_commit/pre_commit_mypy.sh files: ^docs/.*\.py$ exclude: rtd-deprecation require_serial: false - id: pylint name: Run pylint for main code language: system - entry: "./scripts/ci/pre_commit/pre_commit_pylint.sh" + entry: ./scripts/ci/pre_commit/pre_commit_pylint.sh files: \.py$ exclude: ^scripts/.*\.py$|^dev|^provider_packages|^chart|^tests|^kubernetes_tests pass_filenames: true @@ -603,21 +603,21 @@ repos: - id: pylint name: Run pylint for tests language: system - entry: "env PYLINTRC=pylintrc-tests ./scripts/ci/pre_commit/pre_commit_pylint.sh" + entry: env PYLINTRC=pylintrc-tests ./scripts/ci/pre_commit/pre_commit_pylint.sh files: ^tests/.*\.py$ pass_filenames: true require_serial: true - id: pylint name: Run pylint for helm chart tests language: system - entry: "env PYLINTRC=pylintrc-tests ./scripts/ci/pre_commit/pre_commit_pylint.sh" + entry: env PYLINTRC=pylintrc-tests ./scripts/ci/pre_commit/pre_commit_pylint.sh files: ^chart/.*\.py$ pass_filenames: true require_serial: true - id: flake8 name: Run flake8 language: system - entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh" + entry: ./scripts/ci/pre_commit/pre_commit_flake8.sh files: \.py$ pass_filenames: true - id: ui-lint @@ -625,12 +625,12 @@ repos: language: node 'types_or': [javascript, tsx, ts] files: ^airflow/ui/ - entry: "scripts/ci/static_checks/eslint.sh" + entry: ./scripts/ci/static_checks/eslint.sh pass_filenames: false - id: bats-in-container-tests name: Run in container bats tests language: system - entry: "./scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh" - files: ^tests/bats/in_container/.*.bats$|^scripts/in_container/.*sh + entry: ./scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh + files: ^tests/bats/in_container/.*\.bats$|^scripts/in_container/.*sh pass_filenames: false ## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE diff --git a/BREEZE.rst b/BREEZE.rst index 68cc9b849531c..2d6a5070a5c2b 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -390,7 +390,7 @@ tmux session with four panes: - one to monitor the scheduler, - one for the webserver, - - one monitors and compiles Javascript files, + - one monitors and compiles JavaScript files, - one with a shell for additional commands. Managing Prod environment (with ``--production-image`` flag): @@ -541,7 +541,7 @@ For all development tasks, unit tests, integration tests, and static code checks **CI image** maintained on the DockerHub in the ``apache/airflow`` repository. This Docker image contains a lot of test-related packages (size of ~1GB). Its tag follows the pattern of ``-python-ci`` -(for example, ``apache/airflow:master-python3.6-ci`` or ``apache/airflow:v1-10-test-python3.6-ci``). +(for example, ``apache/airflow:master-python3.6-ci`` or ``apache/airflow:v2-0-test-python3.6-ci``). The image is built using the ``_ Dockerfile. The CI image is built automatically as needed, however it can be rebuilt manually with @@ -638,7 +638,7 @@ The **Production image** is also maintained on the DockerHub in the ``apache/airflow`` repository. This Docker image (and Dockerfile) contains size-optimised Airflow installation with selected extras and dependencies. Its tag follows the pattern of ``-python`` (for example, ``apache/airflow:master-python3.6`` -or ``apache/airflow:v1-10-test-python3.6``). +or ``apache/airflow:v2-0-test-python3.6``). However in many cases you want to add your own custom version of the image - with added apt dependencies, python dependencies, additional Airflow extras. Breeze's ``build-image`` command helps to build your own, @@ -693,56 +693,6 @@ Same as above but uses python 3.7. -Building Production images for 1.10 Airflow versions ----------------------------------------------------- - -With Breeze you can also use the master Dockerfile to build custom images for released Airflow versions. -This works in the same way as building production image from master, but you need to add additional switch -``--install-airflow-version``. You should pass version of airflow (as released in PyPI). It can be used -to install both released versions and release candidates. Similarly as in case of master images, -we can pass additional extras/dependencies to install via the additional flags. - -.. code-block:: bash - - ./breeze build-image --production-image --additional-extras "jira" --install-airflow-version="1.10.11" - -Builds airflow image with released Airflow version 1.10.11 and additional extra "jira" added. - -.. code-block:: bash - - ./breeze build-image --production-image --install-airflow-version="1.10.11rc2" - -Builds airflow image with released Airflow version 1.10.11rc2. - - -You can also build airflow directly from GitHub source code - by providing Git Reference via -``--install-airflow-reference``. The reference can be a branch name, tag name, or commit hash. This -is useful mostly for testing. - -.. code-block:: bash - - ./breeze build-image --production-image --install-airflow-reference="v1-10-test" - -This Builds airflow image from the current ``v1-10-test`` branch of Airflow. - -.. code-block:: bash - - ./breeze build-image --production-image \ - --install-airflow-reference="0d91fcf725f69e10f0969ca36f9e38e1d74110d0" - -This Builds airflow image from the ``0d91fcf725f69e10f0969ca36f9e38e1d74110d0`` commit hash on -GitHub. - -.. raw:: html - -
- - Airflow Breeze - Building Production images for 1.10 Airflow versions - -
- - Running static checks --------------------- @@ -823,8 +773,7 @@ Generating constraints ---------------------- Whenever setup.py gets modified, the CI master job will re-generate constraint files. Those constraint -files are stored in separated orphan branches: ``constraints-master``, ``constraints-2-0`` -and ``constraints-1-10``. +files are stored in separated orphan branches: ``constraints-master``, ``constraints-2-0``. Those are constraint files as described in detail in the ``_ contributing documentation. @@ -1319,21 +1268,18 @@ This is the current syntax for `./breeze <./breeze>`_: -p, --python PYTHON_MAJOR_MINOR_VERSION Python version used for the image. This is always major/minor version. - Note that versions 2.7 and 3.5 are only valid when installing Airflow 1.10 with - --install-airflow-version or --install-airflow-reference flags. - One of: - 2.7 3.5 3.6 3.7 3.8 + 3.6 3.7 3.8 -a, --install-airflow-version INSTALL_AIRFLOW_VERSION - Uses differen version of Airflow when building PROD image. + Uses different version of Airflow when building PROD image. - 2.0.2 2.0.1 2.0.0 1.10.15 1.10.14 wheel sdist + 2.0.2 2.0.1 2.0.0 wheel sdist -t, --install-airflow-reference INSTALL_AIRFLOW_REFERENCE Installs Airflow directly from reference in GitHub when building PROD image. - This can be a GitHub branch like master or v1-10-test, or a tag like 2.0.0a1. + This can be a GitHub branch like master or v2-0-test, or a tag like 2.0.0a1. --installation-method INSTALLATION_METHOD Method of installing Airflow in PROD image - either from the sources ('.') @@ -1564,12 +1510,9 @@ This is the current syntax for `./breeze <./breeze>`_: -p, --python PYTHON_MAJOR_MINOR_VERSION Python version used for the image. This is always major/minor version. - Note that versions 2.7 and 3.5 are only valid when installing Airflow 1.10 with - --install-airflow-version or --install-airflow-reference flags. - One of: - 2.7 3.5 3.6 3.7 3.8 + 3.6 3.7 3.8 -I, --production-image Use production image for entering the environment and builds (not for tests). @@ -1611,7 +1554,7 @@ This is the current syntax for `./breeze <./breeze>`_: Generates pinned constraint files with all extras from setup.py. Those files are generated in files folder - separate files for different python version. Those constraint files when - pushed to orphan constraints-master, constraints-2-0 and constraints-1-10 branches are used + pushed to orphan constraints-master, constraints-2-0 branches are used to generate repeatable CI builds as well as run repeatable production image builds and upgrades when you want to include installing or updating some of the released providers released at the time particular airflow version was released. You can use those @@ -1634,12 +1577,9 @@ This is the current syntax for `./breeze <./breeze>`_: -p, --python PYTHON_MAJOR_MINOR_VERSION Python version used for the image. This is always major/minor version. - Note that versions 2.7 and 3.5 are only valid when installing Airflow 1.10 with - --install-airflow-version or --install-airflow-reference flags. - One of: - 2.7 3.5 3.6 3.7 3.8 + 3.6 3.7 3.8 -v, --verbose Show verbose information about executed docker, kind, kubectl, helm commands. Useful for @@ -1760,12 +1700,9 @@ This is the current syntax for `./breeze <./breeze>`_: -p, --python PYTHON_MAJOR_MINOR_VERSION Python version used for the image. This is always major/minor version. - Note that versions 2.7 and 3.5 are only valid when installing Airflow 1.10 with - --install-airflow-version or --install-airflow-reference flags. - One of: - 2.7 3.5 3.6 3.7 3.8 + 3.6 3.7 3.8 #################################################################################################### @@ -1779,11 +1716,19 @@ This is the current syntax for `./breeze <./breeze>`_: Prepares airflow packages (sdist and wheel) in dist folder. Note that prepare-provider-packages command cleans up the dist folder, so if you want also to generate provider packages, make sure you run prepare-provider-packages first, - and prepare-airflow-packages second. + and prepare-airflow-packages second. You can specify optional + --version-suffix-for-svn flag to generate rc candidate packages to upload to SVN or + --version-suffix-for-pypi flag to generate rc candidates for PyPI packages. You can also + provide both suffixes in case you prepare alpha/beta versions. The packages are prepared in + dist folder - General form: + Examples: - 'breeze prepare-airflow-packages + 'breeze prepare-airflow-packages --package-format wheel' or + 'breeze prepare-airflow-packages --version-suffix-for-svn rc1' or + 'breeze prepare-airflow-packages --version-suffix-for-pypi rc1' + 'breeze prepare-airflow-packages --version-suffix-for-pypi a1 + --version-suffix-for-svn a1' Flags: @@ -1797,6 +1742,14 @@ This is the current syntax for `./breeze <./breeze>`_: Default: both + -S, --version-suffix-for-pypi SUFFIX + Adds optional suffix to the version in the generated provider package. It can be used + to generate rc1/rc2 ... versions of the packages to be uploaded to PyPI. + + -N, --version-suffix-for-svn SUFFIX + Adds optional suffix to the generated names of package. It can be used to generate + rc1/rc2 ... versions of the packages to be uploaded to SVN. + -v, --verbose Show verbose information about executed docker, kind, kubectl, helm commands. Useful for debugging - when you run breeze with --verbose flags you will be able to see the commands @@ -1848,16 +1801,13 @@ This is the current syntax for `./breeze <./breeze>`_: airflow is just removed. In this case airflow package should be added to dist folder and --use-packages-from-dist flag should be used. - 2.0.2 2.0.1 2.0.0 1.10.15 1.10.14 wheel sdist none + 2.0.2 2.0.1 2.0.0 wheel sdist none --use-packages-from-dist In CI image, if specified it will look for packages placed in dist folder and it will install the packages after entering the image. This is useful for testing provider packages. - --no-rbac-ui - Disables RBAC UI when Airflow 1.10.* is installed. - --load-example-dags Include Airflow example dags. @@ -1950,12 +1900,9 @@ This is the current syntax for `./breeze <./breeze>`_: -p, --python PYTHON_MAJOR_MINOR_VERSION Python version used for the image. This is always major/minor version. - Note that versions 2.7 and 3.5 are only valid when installing Airflow 1.10 with - --install-airflow-version or --install-airflow-reference flags. - One of: - 2.7 3.5 3.6 3.7 3.8 + 3.6 3.7 3.8 -b, --backend BACKEND Backend to use for tests - it determines which database is used. @@ -2017,12 +1964,9 @@ This is the current syntax for `./breeze <./breeze>`_: -p, --python PYTHON_MAJOR_MINOR_VERSION Python version used for the image. This is always major/minor version. - Note that versions 2.7 and 3.5 are only valid when installing Airflow 1.10 with - --install-airflow-version or --install-airflow-reference flags. - One of: - 2.7 3.5 3.6 3.7 3.8 + 3.6 3.7 3.8 -F, --force-build-images Forces building of the local docker images. The images are rebuilt @@ -2419,12 +2363,9 @@ This is the current syntax for `./breeze <./breeze>`_: -p, --python PYTHON_MAJOR_MINOR_VERSION Python version used for the image. This is always major/minor version. - Note that versions 2.7 and 3.5 are only valid when installing Airflow 1.10 with - --install-airflow-version or --install-airflow-reference flags. - One of: - 2.7 3.5 3.6 3.7 3.8 + 3.6 3.7 3.8 **************************************************************************************************** Choose backend to run for Airflow @@ -2531,6 +2472,14 @@ This is the current syntax for `./breeze <./breeze>`_: Default: v3.2.4 + --executor EXECUTOR + Executor to use in a kubernetes cluster. + One of: + + KubernetesExecutor CeleryExecutor LocalExecutor CeleryKubernetesExecutor + + Default: KubernetesExecutor + **************************************************************************************************** Manage mounting local files @@ -2554,13 +2503,13 @@ This is the current syntax for `./breeze <./breeze>`_: Install different Airflow version during PROD image build -a, --install-airflow-version INSTALL_AIRFLOW_VERSION - Uses differen version of Airflow when building PROD image. + Uses different version of Airflow when building PROD image. - 2.0.2 2.0.1 2.0.0 1.10.15 1.10.14 wheel sdist + 2.0.2 2.0.1 2.0.0 wheel sdist -t, --install-airflow-reference INSTALL_AIRFLOW_REFERENCE Installs Airflow directly from reference in GitHub when building PROD image. - This can be a GitHub branch like master or v1-10-test, or a tag like 2.0.0a1. + This can be a GitHub branch like master or v2-0-test, or a tag like 2.0.0a1. --installation-method INSTALLATION_METHOD Method of installing Airflow in PROD image - either from the sources ('.') @@ -2584,16 +2533,13 @@ This is the current syntax for `./breeze <./breeze>`_: airflow is just removed. In this case airflow package should be added to dist folder and --use-packages-from-dist flag should be used. - 2.0.2 2.0.1 2.0.0 1.10.15 1.10.14 wheel sdist none + 2.0.2 2.0.1 2.0.0 wheel sdist none --use-packages-from-dist In CI image, if specified it will look for packages placed in dist folder and it will install the packages after entering the image. This is useful for testing provider packages. - --no-rbac-ui - Disables RBAC UI when Airflow 1.10.* is installed. - **************************************************************************************************** Credentials diff --git a/CI.rst b/CI.rst index e7da352f4858f..9bee4c3689cc3 100644 --- a/CI.rst +++ b/CI.rst @@ -21,7 +21,7 @@ CI Environment ============== Continuous Integration is important component of making Apache Airflow robust and stable. We are running -a lot of tests for every pull request, for master and v1-10-test branches and regularly as CRON jobs. +a lot of tests for every pull request, for master and v2-0-test branches and regularly as CRON jobs. Our execution environment for CI is `GitHub Actions `_. GitHub Actions (GA) are very well integrated with GitHub code and Workflow and it has evolved fast in 2019/202 to become @@ -106,7 +106,7 @@ Default is the GitHub Package Registry one. The Pull Request forks have no acces auto-detect the registry used when they wait for the images. You can interact with the GitHub Registry images (pull/push) via `Breeze `_ - you can -pass ``--github-registry`` flag wih either ``docker.pkg.github.com`` for GitHub Package Registry or +pass ``--github-registry`` flag with either ``docker.pkg.github.com`` for GitHub Package Registry or ``ghcr.io`` for GitHub Container Registry and pull/push operations will be performed using the chosen registry, using appropriate naming convention. This allows building and pushing the images locally by committers who have access to push/pull those images. @@ -307,7 +307,7 @@ You can use those variables when you try to reproduce the build locally. | | | | | tested set of dependency constraints | | | | | | stored in separated "orphan" branches | | | | | | of the airflow repository | -| | | | | ("constraints-master, "constraints-1-10") | +| | | | | ("constraints-master, "constraints-2-0") | | | | | | but when this flag is set to anything but false | | | | | | (for example commit SHA), they are not used | | | | | | used and "eager" upgrade strategy is used | diff --git a/Dockerfile b/Dockerfile index b285fe47d4b6c..23b365dfd3110 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,7 @@ ARG AIRFLOW_GID="50000" ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" -ARG AIRFLOW_PIP_VERSION=21.1 +ARG AIRFLOW_PIP_VERSION=21.1.1 # By default PIP has progress bar but you can disable it. ARG PIP_PROGRESS_BAR="on" @@ -104,7 +104,7 @@ ARG DEV_APT_DEPS="\ yarn" ARG ADDITIONAL_DEV_APT_DEPS="" ARG DEV_APT_COMMAND="\ - curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ + curl --fail --location https://deb.nodesource.com/setup_14.x | bash - \ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" ARG ADDITIONAL_DEV_APT_COMMAND="echo" diff --git a/Dockerfile.ci b/Dockerfile.ci index 04f8edf3c9aa7..ded0a37772853 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -21,7 +21,7 @@ FROM ${PYTHON_BASE_IMAGE} as main SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" -ARG AIRFLOW_VERSION="2.0.0.dev0" +ARG AIRFLOW_VERSION="2.1.0.dev0" # By increasing this number we can do force build of all dependencies ARG DEPENDENCIES_EPOCH_NUMBER="6" @@ -45,7 +45,7 @@ RUN apt-get update \ ARG ADDITIONAL_DEV_APT_DEPS="" ARG DEV_APT_COMMAND="\ - curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ + curl --fail --location https://deb.nodesource.com/setup_14.x | bash - \ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" ARG ADDITIONAL_DEV_APT_COMMAND="" diff --git a/IMAGES.rst b/IMAGES.rst index 34332e438c9a0..7215871b25cd7 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -69,8 +69,8 @@ The images are named as follows: where: * ``BRANCH_OR_TAG`` - branch or tag used when creating the image. Examples: ``master``, - ``v2-0-test``, ``v1-10-test``, ``2.0.0``. The ``master``, ``v1-10-test`` ``v2-0-test`` labels are - built from branches so they change over time. The ``1.10.*`` and ``2.*`` labels are built from git tags + ``v2-0-test``, ``2.0.0``. The ``master``, ``v2-0-test`` labels are + built from branches so they change over time. The ````2.*`` labels are built from git tags and they are "fixed" once built. * ``PYTHON_MAJOR_MINOR_VERSION`` - version of Python used to build the image. Examples: ``3.6``, ``3.7``, ``3.8`` @@ -562,7 +562,6 @@ The following build arguments (``--build-arg`` in docker build command) can be u | | | used. By default it is set to | | | | ``constraints-master`` but can be | | | | ``constraints-2-0`` for 2.0.* versions | -| | | ``constraints-1-10`` for 1.10.* versions | | | | or it could point to specific version | | | | for example ``constraints-2.0.0`` | | | | is empty, it is auto-detected | diff --git a/INSTALL b/INSTALL index 919c4f540d83a..013755440278f 100644 --- a/INSTALL +++ b/INSTALL @@ -71,7 +71,7 @@ This is useful if you want to develop providers: pip install -e . \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-master/constraints-3.6.txt" -You can als skip installing provider packages from PyPI by setting INSTALL_PROVIDERS_FROM_SOURCE to "true". +You can also skip installing provider packages from PyPI by setting INSTALL_PROVIDERS_FROM_SOURCE to "true". In this case Airflow will be installed in non-editable mode with all providers installed from the sources. Additionally `provider.yaml` files will also be copied to providers folders which will make the providers discoverable by Airflow even if they are not installed from packages in this case. diff --git a/README.md b/README.md index e2f6015452fe3..2209b04d4f0c8 100644 --- a/README.md +++ b/README.md @@ -144,9 +144,8 @@ our dependencies as open as possible (in `setup.py`) so users can install differ if needed. This means that from time to time plain `pip install apache-airflow` will not work or will produce unusable Airflow installation. -In order to have repeatable installation, however, introduced in **Airflow 1.10.10** and updated in -**Airflow 1.10.12** we also keep a set of "known-to-be-working" constraint files in the -orphan `constraints-master`, `constraints-2-0` and `constraints-1-10` branches. We keep those "known-to-be-working" +In order to have repeatable installation, however, we also keep a set of "known-to-be-working" constraint +files in the orphan `constraints-master`, `constraints-2-0` branches. We keep those "known-to-be-working" constraints files separately per major/minor Python version. You can use them as constraint files when installing Airflow from PyPI. Note that you have to specify correct Airflow tag/version/branch and Python versions in the URL. diff --git a/TESTING.rst b/TESTING.rst index e306b8db94bcf..dd53a0629fd90 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -446,7 +446,7 @@ test in parallel. This way we can decrease the time of running all tests in self .. note:: We need to split tests manually into separate suites rather than utilise - ``pytest-xdist`` or ``pytest-parallel`` which could ba a simpler and much more "native" parallelization + ``pytest-xdist`` or ``pytest-parallel`` which could be a simpler and much more "native" parallelization mechanism. Unfortunately, we cannot utilise those tools because our tests are not truly ``unit`` tests that can run in parallel. A lot of our tests rely on shared databases - and they update/reset/cleanup the databases while they are executing. They are also exercising features of the Database such as locking which @@ -601,6 +601,17 @@ The deploy command performs those steps: 5. Applies the volumes.yaml to get the volumes deployed to ``default`` namespace - this is where KubernetesExecutor starts its pods. +You can also specify a different executor by providing the ``--executor`` optional argument: + +.. code-block:: bash + + ./breeze kind-cluster deploy --executor CeleryExecutor + +Note that when you specify the ``--executor`` option, it becomes the default. Therefore, every other operations +on ``./breeze kind-cluster`` will default to using this executor. To change that, use the ``--executor`` option on the +subsequent commands too. + + Running tests with Kubernetes Cluster ------------------------------------- @@ -622,6 +633,12 @@ Running Kubernetes tests via breeze: ./breeze kind-cluster test ./breeze kind-cluster test -- TEST TEST [TEST ...] +Optionally add ``--executor``: + +.. code-block:: bash + + ./breeze kind-cluster test --executor CeleryExecutor + ./breeze kind-cluster test -- TEST TEST [TEST ...] --executor CeleryExecutor Entering shell with Kubernetes Cluster -------------------------------------- @@ -645,6 +662,12 @@ You can enter the shell via those scripts ./breeze kind-cluster shell +Optionally add ``--executor``: + +.. code-block:: bash + + ./breeze kind-cluster shell --executor CeleryExecutor + K9s CLI - debug Kubernetes in style! ------------------------------------ diff --git a/UPDATING.md b/UPDATING.md index 24435b5e0db9f..59aa7724e3d7a 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -71,6 +71,17 @@ https://developers.google.com/style/inclusive-documentation --> +### `@apply_default` decorator isn't longer necessary + +This decorator is now automatically added to all operators via the metaclass on BaseOperator + +### Change the configuration options for field masking + +We've improved masking for sensitive data in Web UI and logs. As part of it, the following configurations have been changed: + +* `hide_sensitive_variable_fields` option in `admin` section has been replaced by `hide_sensitive_var_conn_fields` section in `core` section, +* `sensitive_variable_fields` option in `admin` section has been replaced by `sensitive_var_conn_names` section in `core` section. + ### Deprecated PodDefaults and add_xcom_sidecar in airflow.kubernetes.pod_generator We have moved PodDefaults from `airflow.kubernetes.pod_generator.PodDefaults` to @@ -1701,7 +1712,7 @@ Rename `sign_in` function to `get_conn`. #### `airflow.providers.apache.pinot.hooks.pinot.PinotAdminHook.create_segment` -Rename parameter name from ``format`` to ``segment_format`` in PinotAdminHook function create_segment fro pylint compatible +Rename parameter name from ``format`` to ``segment_format`` in PinotAdminHook function create_segment for pylint compatible #### `airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_partitions` diff --git a/airflow/api/common/experimental/mark_tasks.py b/airflow/api/common/experimental/mark_tasks.py index 04215dc41e95b..7612270b921f5 100644 --- a/airflow/api/common/experimental/mark_tasks.py +++ b/airflow/api/common/experimental/mark_tasks.py @@ -161,7 +161,7 @@ def get_all_dag_task_query(dag, session, state, task_ids, confirmed_dates): def get_subdag_runs(dag, session, state, task_ids, commit, confirmed_dates): """Go through subdag operators and create dag runs. We will only work - within the scope of the subdag. We wont propagate to the parent dag, + within the scope of the subdag. We won't propagate to the parent dag, but we will propagate from parent to subdag. """ dags = [dag] diff --git a/airflow/api_connexion/endpoints/provider_endpoint.py b/airflow/api_connexion/endpoints/provider_endpoint.py new file mode 100644 index 0000000000000..844dcd301495c --- /dev/null +++ b/airflow/api_connexion/endpoints/provider_endpoint.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re +from typing import Dict, List + +from airflow.api_connexion import security +from airflow.api_connexion.schemas.provider_schema import ProviderCollection, provider_collection_schema +from airflow.providers_manager import ProviderInfo, ProvidersManager +from airflow.security import permissions + + +def _remove_rst_syntax(value: str) -> str: + return re.sub("[`_<>]", "", value.strip(" \n.")) + + +def _provider_mapper(provider: ProviderInfo) -> Dict: + return { + "package_name": provider[1]["package-name"], + "description": _remove_rst_syntax(provider[1]["description"]), + "version": provider[0], + } + + +@security.requires_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_PROVIDER)]) +def get_providers(): + """Get providers""" + providers_info: List[ProviderInfo] = list(ProvidersManager().providers.values()) + providers = [_provider_mapper(d) for d in providers_info] + total_entries = len(providers) + return provider_collection_schema.dump( + ProviderCollection(providers=providers, total_entries=total_entries) + ) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 2fd58ea326955..72c46d665cb37 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -133,6 +133,7 @@ info: |-|-| | v2.0 | Initial release | | v2.0.2 | Added /plugins endpoint | + | v2.1 | New providers endpoint | # Trying the API @@ -849,6 +850,26 @@ paths: '404': $ref: '#/components/responses/NotFound' + /providers: + get: + summary: List providers + x-openapi-router-controller: airflow.api_connexion.endpoints.provider_endpoint + operationId: get_providers + tags: [Provider] + responses: + '200': + description: List of providers. + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/ProviderCollection' + - $ref: '#/components/schemas/CollectionInfo' + '401': + $ref: '#/components/responses/Unauthenticated' + '403': + $ref: '#/components/responses/PermissionDenied' + /dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances: parameters: - $ref: '#/components/parameters/DAGID' @@ -1860,7 +1881,7 @@ components: description: Log of user operations via CLI or Web UI. properties: event_log_id: - description: The evnet log ID + description: The event log ID type: integer readOnly: true when: @@ -2016,6 +2037,29 @@ components: - $ref: '#/components/schemas/CollectionInfo' + Provider: + description: The provider + type: object + properties: + package_name: + type: string + description: The package name of the provider. + description: + type: string + description: The description of the provider. + version: + type: string + description: The version of the provider. + + ProviderCollection: + type: object + properties: + providers: + type: array + items: + $ref: '#/components/schemas/Provider' + + SLAMiss: type: object properties: @@ -3390,6 +3434,7 @@ tags: - name: ImportError - name: Monitoring - name: Pool + - name: Provider - name: TaskInstance - name: Variable - name: XCom diff --git a/tests/dags/test_backfill_pooled_tasks.py b/airflow/api_connexion/schemas/provider_schema.py similarity index 54% rename from tests/dags/test_backfill_pooled_tasks.py rename to airflow/api_connexion/schemas/provider_schema.py index 77ebd22360100..df0ddd5e33821 100644 --- a/tests/dags/test_backfill_pooled_tasks.py +++ b/airflow/api_connexion/schemas/provider_schema.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,22 +15,32 @@ # specific language governing permissions and limitations # under the License. +from typing import List, NamedTuple + +from marshmallow import Schema, fields + + +class ProviderSchema(Schema): + """Provider schema""" + + package_name = fields.String(required=True) + description = fields.String(required=True) + version = fields.String(required=True) + + +class ProviderCollection(NamedTuple): + """List of Providers""" + + providers: List[ProviderSchema] + total_entries: int + + +class ProviderCollectionSchema(Schema): + """Provider Collection schema""" + + providers = fields.List(fields.Nested(ProviderSchema)) + total_entries = fields.Int() + -""" -DAG designed to test what happens when a DAG with pooled tasks is run -by a BackfillJob. -Addresses issue #1225. -""" -from datetime import datetime - -from airflow.models import DAG -from airflow.operators.dummy import DummyOperator - -dag = DAG(dag_id='test_backfill_pooled_task_dag') -task = DummyOperator( - task_id='test_backfill_pooled_task', - dag=dag, - pool='test_backfill_pooled_task_pool', - owner='airflow', - start_date=datetime(2016, 2, 1), -) +provider_collection_schema = ProviderCollectionSchema() +provider_schema = ProviderSchema() diff --git a/airflow/cli/commands/task_command.py b/airflow/cli/commands/task_command.py index 373f933d4cfa5..dc38b87835c3d 100644 --- a/airflow/cli/commands/task_command.py +++ b/airflow/cli/commands/task_command.py @@ -202,6 +202,8 @@ def task_run(args, dag=None): conf.read_dict(conf_dict, source=args.cfg_path) settings.configure_vars() + settings.MASK_SECRETS_IN_LOGS = True + # IMPORTANT, have to use the NullPool, otherwise, each "run" command may leave # behind multiple open sleeping connections while heartbeating, which could # easily exceed the database connection limit when @@ -357,6 +359,9 @@ def task_test(args, dag=None): # We want to log output from operators etc to show up here. Normally # airflow.task would redirect to a file, but here we want it to propagate # up to the normal airflow handler. + + settings.MASK_SECRETS_IN_LOGS = True + handlers = logging.getLogger('airflow.task').handlers already_has_stream_handler = False for handler in handlers: diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py index ad64078806fb4..3c8ffb844bea6 100644 --- a/airflow/config_templates/airflow_local_settings.py +++ b/airflow/config_templates/airflow_local_settings.py @@ -64,23 +64,31 @@ 'class': COLORED_FORMATTER_CLASS if COLORED_LOG else 'logging.Formatter', }, }, + 'filters': { + 'mask_secrets': { + '()': 'airflow.utils.log.secrets_masker.SecretsMasker', + }, + }, 'handlers': { 'console': { 'class': 'airflow.utils.log.logging_mixin.RedirectStdHandler', 'formatter': 'airflow_coloured', 'stream': 'sys.stdout', + 'filters': ['mask_secrets'], }, 'task': { 'class': 'airflow.utils.log.file_task_handler.FileTaskHandler', 'formatter': 'airflow', 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER), 'filename_template': FILENAME_TEMPLATE, + 'filters': ['mask_secrets'], }, 'processor': { 'class': 'airflow.utils.log.file_processor_handler.FileProcessorHandler', 'formatter': 'airflow', 'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER), 'filename_template': PROCESSOR_FILENAME_TEMPLATE, + 'filters': ['mask_secrets'], }, }, 'loggers': { @@ -93,6 +101,7 @@ 'handlers': ['task'], 'level': LOG_LEVEL, 'propagate': False, + 'filters': ['mask_secrets'], }, 'flask_appbuilder': { 'handler': ['console'], @@ -103,6 +112,7 @@ 'root': { 'handlers': ['console'], 'level': LOG_LEVEL, + 'filters': ['mask_secrets'], }, } diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 8f66df9676d4b..54b43b1443455 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -243,7 +243,7 @@ version_added: ~ type: float example: ~ - default: "30" + default: "30.0" - name: dagbag_import_error_tracebacks description: | Should a traceback be shown in the UI for dagbag import errors, @@ -414,6 +414,23 @@ type: integer example: ~ default: "3" + - name: hide_sensitive_var_conn_fields + description: | + Hide sensitive Variables or Connection extra json keys from UI and task logs when set to True + + (Connection passwords are always hidden in logs) + version_added: ~ + type: boolean + example: ~ + default: "True" + - name: sensitive_var_conn_names + description: | + A comma-separated list of extra sensitive keywords to look for in variables names or connection's + extra JSON. + version_added: ~ + type: string + example: ~ + default: "" - name: logging description: ~ @@ -1911,23 +1928,6 @@ type: string example: ~ default: "v3" -- name: admin - description: ~ - options: - - name: hide_sensitive_variable_fields - description: | - UI to hide sensitive variable fields when set to True - version_added: ~ - type: string - example: ~ - default: "True" - - name: sensitive_variable_fields - description: | - A comma-separated list of sensitive keywords to look for in variables names. - version_added: ~ - type: string - example: ~ - default: "" - name: elasticsearch description: ~ options: diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 0f906063e6f52..d409801d0a428 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -148,7 +148,7 @@ fernet_key = {FERNET_KEY} donot_pickle = True # How long before timing out a python file import -dagbag_import_timeout = 30 +dagbag_import_timeout = 30.0 # Should a traceback be shown in the UI for dagbag import errors, # instead of just the exception message @@ -237,6 +237,15 @@ lazy_discover_providers = True # Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. max_db_retries = 3 +# Hide sensitive Variables or Connection extra json keys from UI and task logs when set to True +# +# (Connection passwords are always hidden in logs) +hide_sensitive_var_conn_fields = True + +# A comma-separated list of extra sensitive keywords to look for in variables names or connection's +# extra JSON. +sensitive_var_conn_names = + [logging] # The folder where airflow should store its log files # This path must be absolute @@ -949,13 +958,6 @@ keytab = airflow.keytab [github_enterprise] api_rev = v3 -[admin] -# UI to hide sensitive variable fields when set to True -hide_sensitive_variable_fields = True - -# A comma-separated list of sensitive keywords to look for in variables names. -sensitive_variable_fields = - [elasticsearch] # Elasticsearch host host = diff --git a/airflow/config_templates/default_test.cfg b/airflow/config_templates/default_test.cfg index 5b647aa90d5a5..2cbdb16446cbc 100644 --- a/airflow/config_templates/default_test.cfg +++ b/airflow/config_templates/default_test.cfg @@ -113,10 +113,6 @@ scheduler_zombie_task_threshold = 300 dag_dir_list_interval = 0 max_tis_per_query = 512 -[admin] -hide_sensitive_variable_fields = True -sensitive_variable_fields = - [elasticsearch] host = log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}} diff --git a/airflow/configuration.py b/airflow/configuration.py index 20a03a0cccc5b..4420ddaf278e3 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -160,6 +160,8 @@ class AirflowConfigParser(ConfigParser): # pylint: disable=too-many-ancestors ('metrics', 'statsd_custom_client_path'): ('scheduler', 'statsd_custom_client_path', '2.0.0'), ('scheduler', 'parsing_processes'): ('scheduler', 'max_threads', '1.10.14'), ('operators', 'default_queue'): ('celery', 'default_queue', '2.1.0'), + ('core', 'hide_sensitive_var_conn_fields'): ('admin', 'hide_sensitive_variable_fields', '2.1.0'), + ('core', 'sensitive_var_conn_names'): ('admin', 'sensitive_variable_fields', '2.1.0'), } # A mapping of old default values that we want to change and warn the user diff --git a/airflow/contrib/operators/adls_to_gcs.py b/airflow/contrib/operators/adls_to_gcs.py index 0bdebfc14c455..0497d4c259023 100644 --- a/airflow/contrib/operators/adls_to_gcs.py +++ b/airflow/contrib/operators/adls_to_gcs.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.transfers.adls_to_gcs.ADLSToGCSOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/awsbatch_operator.py b/airflow/contrib/operators/awsbatch_operator.py index d799ea9be794e..0d1c5b0bd3d5d 100644 --- a/airflow/contrib/operators/awsbatch_operator.py +++ b/airflow/contrib/operators/awsbatch_operator.py @@ -53,7 +53,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.amazon.aws.operators.batch.AwsBatchOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/bigquery_operator.py b/airflow/contrib/operators/bigquery_operator.py index ca3f1fdbdf115..ca9a84580e915 100644 --- a/airflow/contrib/operators/bigquery_operator.py +++ b/airflow/contrib/operators/bigquery_operator.py @@ -51,6 +51,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.bigquery.BigQueryExecuteQueryOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/bigquery_table_delete_operator.py b/airflow/contrib/operators/bigquery_table_delete_operator.py index 0a2603998b96e..13822a1844409 100644 --- a/airflow/contrib/operators/bigquery_table_delete_operator.py +++ b/airflow/contrib/operators/bigquery_table_delete_operator.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/bigquery_to_gcs.py b/airflow/contrib/operators/bigquery_to_gcs.py index d0c10ed14f63a..702171187e885 100644 --- a/airflow/contrib/operators/bigquery_to_gcs.py +++ b/airflow/contrib/operators/bigquery_to_gcs.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.transfers.bigquery_to_gcs.BigQueryToGCSOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/cassandra_to_gcs.py b/airflow/contrib/operators/cassandra_to_gcs.py index 0b604ce333e28..bb4b244f0d4cf 100644 --- a/airflow/contrib/operators/cassandra_to_gcs.py +++ b/airflow/contrib/operators/cassandra_to_gcs.py @@ -42,6 +42,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.transfers.cassandra_to_gcs.CassandraToGCSOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/dataflow_operator.py b/airflow/contrib/operators/dataflow_operator.py index cedf77c9c5ea0..d2e445add02f1 100644 --- a/airflow/contrib/operators/dataflow_operator.py +++ b/airflow/contrib/operators/dataflow_operator.py @@ -43,7 +43,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dataflow.DataflowCreateJavaJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.dataflow.DataflowCreatePythonJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -77,6 +77,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.dataflow.DataflowTemplatedJobStartOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py index 76bde2af85298..b655ce630142a 100644 --- a/airflow/contrib/operators/dataproc_operator.py +++ b/airflow/contrib/operators/dataproc_operator.py @@ -52,7 +52,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dataproc.DataprocCreateClusterOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -68,7 +68,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dataproc.DataprocDeleteClusterOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -84,7 +84,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dataproc.DataprocScaleClusterOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -101,7 +101,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitHadoopJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -118,7 +118,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitHiveJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -134,7 +134,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dataproc.DataprocJobBaseOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -150,7 +150,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitPigJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -167,7 +167,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitPySparkJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -184,7 +184,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitSparkJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -201,7 +201,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitSparkSqlJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -220,7 +220,7 @@ def __init__(self, *args, **kwargs): `airflow.providers.google.cloud.operators.dataproc .DataprocInstantiateInlineWorkflowTemplateOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -239,6 +239,6 @@ def __init__(self, *args, **kwargs): `airflow.providers.google.cloud.operators.dataproc .DataprocInstantiateWorkflowTemplateOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/datastore_export_operator.py b/airflow/contrib/operators/datastore_export_operator.py index 1b875581dc2ec..085ee132607da 100644 --- a/airflow/contrib/operators/datastore_export_operator.py +++ b/airflow/contrib/operators/datastore_export_operator.py @@ -40,6 +40,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.datastore.CloudDatastoreExportEntitiesOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/datastore_import_operator.py b/airflow/contrib/operators/datastore_import_operator.py index 97b3bfc34e752..5b15cd23c9516 100644 --- a/airflow/contrib/operators/datastore_import_operator.py +++ b/airflow/contrib/operators/datastore_import_operator.py @@ -40,6 +40,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.datastore.CloudDatastoreImportEntitiesOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/file_to_gcs.py b/airflow/contrib/operators/file_to_gcs.py index a26dc5334e288..227cd74979c2f 100644 --- a/airflow/contrib/operators/file_to_gcs.py +++ b/airflow/contrib/operators/file_to_gcs.py @@ -40,6 +40,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.transfers.local_to_gcs.LocalFilesystemToGCSOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_bigtable_operator.py b/airflow/contrib/operators/gcp_bigtable_operator.py index 7256d64052d14..b07a04c075921 100644 --- a/airflow/contrib/operators/gcp_bigtable_operator.py +++ b/airflow/contrib/operators/gcp_bigtable_operator.py @@ -50,7 +50,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.bigtable.BigtableUpdateClusterOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -66,7 +66,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.bigtable.BigtableCreateInstanceOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -82,7 +82,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.bigtable.BigtableDeleteInstanceOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -98,7 +98,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.bigtable.BigtableCreateTableOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -114,7 +114,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.bigtable.BigtableDeleteTableOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -132,6 +132,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.sensors.bigtable.BigtableTableReplicationCompletedSensor`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_compute_operator.py b/airflow/contrib/operators/gcp_compute_operator.py index ce6f86256c02f..d943fdf5587dd 100644 --- a/airflow/contrib/operators/gcp_compute_operator.py +++ b/airflow/contrib/operators/gcp_compute_operator.py @@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.compute.ComputeEngineBaseOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -64,7 +64,7 @@ def __init__(self, *args, **kwargs): `airflow.providers.google.cloud.operators.compute .ComputeEngineInstanceGroupUpdateManagerTemplateOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -82,7 +82,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.compute .ComputeEngineStartInstanceOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -99,7 +99,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.compute .ComputeEngineStopInstanceOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -116,7 +116,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.compute .ComputeEngineCopyInstanceTemplateOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -133,6 +133,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.compute .ComputeEngineSetMachineTypeOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_container_operator.py b/airflow/contrib/operators/gcp_container_operator.py index ecf42fd79da10..8a26c40a8fe5d 100644 --- a/airflow/contrib/operators/gcp_container_operator.py +++ b/airflow/contrib/operators/gcp_container_operator.py @@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.container.GKECreateClusterOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -62,7 +62,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.container.GKEDeleteClusterOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -78,6 +78,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.container.GKEStartPodOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_dlp_operator.py b/airflow/contrib/operators/gcp_dlp_operator.py index b4a1f3cf274d8..a5f4cae9f4722 100644 --- a/airflow/contrib/operators/gcp_dlp_operator.py +++ b/airflow/contrib/operators/gcp_dlp_operator.py @@ -71,7 +71,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDLPJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -87,7 +87,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -103,7 +103,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobTriggerOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -119,6 +119,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPListDLPJobsOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_function_operator.py b/airflow/contrib/operators/gcp_function_operator.py index de7009cee8908..4ca96c5c8a648 100644 --- a/airflow/contrib/operators/gcp_function_operator.py +++ b/airflow/contrib/operators/gcp_function_operator.py @@ -43,7 +43,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.function.CloudFunctionDeleteFunctionOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -60,6 +60,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.function.CloudFunctionDeployFunctionOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_natural_language_operator.py b/airflow/contrib/operators/gcp_natural_language_operator.py index 38e29659f33ad..8a98ccf6903f0 100644 --- a/airflow/contrib/operators/gcp_natural_language_operator.py +++ b/airflow/contrib/operators/gcp_natural_language_operator.py @@ -50,7 +50,7 @@ def __init__(self, *args, **kwargs): .CloudNaturalLanguageAnalyzeEntitiesOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -71,7 +71,7 @@ def __init__(self, *args, **kwargs): .CloudNaturalLanguageAnalyzeEntitySentimentOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -90,7 +90,7 @@ def __init__(self, *args, **kwargs): .CloudNaturalLanguageAnalyzeSentimentOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -109,6 +109,6 @@ def __init__(self, *args, **kwargs): .CloudNaturalLanguageClassifyTextOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_spanner_operator.py b/airflow/contrib/operators/gcp_spanner_operator.py index 82543d70557b8..b2e50c3a80f6a 100644 --- a/airflow/contrib/operators/gcp_spanner_operator.py +++ b/airflow/contrib/operators/gcp_spanner_operator.py @@ -45,7 +45,7 @@ def __init__(self, *args, **kwargs): warnings.warn( self.__doc__, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs): warnings.warn( self.__doc__, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -75,7 +75,7 @@ def __init__(self, *args, **kwargs): warnings.warn( self.__doc__, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -90,7 +90,7 @@ def __init__(self, *args, **kwargs): warnings.warn( self.__doc__, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -105,7 +105,7 @@ def __init__(self, *args, **kwargs): warnings.warn( self.__doc__, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -120,6 +120,6 @@ def __init__(self, *args, **kwargs): warnings.warn( self.__doc__, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_speech_to_text_operator.py b/airflow/contrib/operators/gcp_speech_to_text_operator.py index 2da4ce981a3cc..499c8ab7f6e01 100644 --- a/airflow/contrib/operators/gcp_speech_to_text_operator.py +++ b/airflow/contrib/operators/gcp_speech_to_text_operator.py @@ -42,6 +42,6 @@ def __init__(self, *args, **kwargs): `airflow.providers.google.cloud.operators.speech_to_text .CloudSpeechToTextRecognizeSpeechOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_sql_operator.py b/airflow/contrib/operators/gcp_sql_operator.py index d68c468b2a8b0..5cbbf8a80a294 100644 --- a/airflow/contrib/operators/gcp_sql_operator.py +++ b/airflow/contrib/operators/gcp_sql_operator.py @@ -46,7 +46,7 @@ class CloudSqlBaseOperator(CloudSQLBaseOperator): """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) @@ -57,7 +57,7 @@ class CloudSqlInstanceCreateOperator(CloudSQLCreateInstanceOperator): """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) @@ -68,7 +68,7 @@ class CloudSqlInstanceDatabaseCreateOperator(CloudSQLCreateInstanceDatabaseOpera """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) @@ -79,7 +79,7 @@ class CloudSqlInstanceDatabaseDeleteOperator(CloudSQLDeleteInstanceDatabaseOpera """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) @@ -90,7 +90,7 @@ class CloudSqlInstanceDatabasePatchOperator(CloudSQLPatchInstanceDatabaseOperato """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) @@ -101,7 +101,7 @@ class CloudSqlInstanceDeleteOperator(CloudSQLDeleteInstanceOperator): """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) @@ -112,7 +112,7 @@ class CloudSqlInstanceExportOperator(CloudSQLExportInstanceOperator): """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) @@ -123,7 +123,7 @@ class CloudSqlInstanceImportOperator(CloudSQLImportInstanceOperator): """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) @@ -134,7 +134,7 @@ class CloudSqlInstancePatchOperator(CloudSQLInstancePatchOperator): """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) @@ -145,5 +145,5 @@ class CloudSqlQueryOperator(CloudSQLExecuteQueryOperator): """ def __init__(self, *args, **kwargs): - warnings.warn(self.__doc__, DeprecationWarning, stacklevel=3) + warnings.warn(self.__doc__, DeprecationWarning, stacklevel=2) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_text_to_speech_operator.py b/airflow/contrib/operators/gcp_text_to_speech_operator.py index aa718238f815e..af5ff2f39d122 100644 --- a/airflow/contrib/operators/gcp_text_to_speech_operator.py +++ b/airflow/contrib/operators/gcp_text_to_speech_operator.py @@ -40,6 +40,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.text_to_speech.CloudTextToSpeechSynthesizeOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_transfer_operator.py b/airflow/contrib/operators/gcp_transfer_operator.py index a939fdc9e5c82..bd7c7bc2cb752 100644 --- a/airflow/contrib/operators/gcp_transfer_operator.py +++ b/airflow/contrib/operators/gcp_transfer_operator.py @@ -56,7 +56,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.data_transfer .CloudDataTransferServiceCreateJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -74,7 +74,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.data_transfer .CloudDataTransferServiceDeleteJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -92,7 +92,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.data_transfer .CloudDataTransferServiceUpdateJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -111,7 +111,7 @@ def __init__(self, *args, **kwargs): .CloudDataTransferServiceCancelOperationOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -129,7 +129,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.data_transfer .CloudDataTransferServiceGetOperationOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -148,7 +148,7 @@ def __init__(self, *args, **kwargs): .CloudDataTransferServicePauseOperationOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -167,7 +167,7 @@ def __init__(self, *args, **kwargs): .CloudDataTransferServiceResumeOperationOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -186,7 +186,7 @@ def __init__(self, *args, **kwargs): .CloudDataTransferServiceListOperationsOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -205,7 +205,7 @@ def __init__(self, *args, **kwargs): .CloudDataTransferServiceGCSToGCSOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -224,6 +224,6 @@ def __init__(self, *args, **kwargs): .CloudDataTransferServiceS3ToGCSOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_translate_speech_operator.py b/airflow/contrib/operators/gcp_translate_speech_operator.py index 8be71a5f9b1bf..f8f8cc56131a6 100644 --- a/airflow/contrib/operators/gcp_translate_speech_operator.py +++ b/airflow/contrib/operators/gcp_translate_speech_operator.py @@ -45,6 +45,6 @@ def __init__(self, *args, **kwargs): `airflow.providers.google.cloud.operators.translate_speech.CloudTranslateSpeechOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcp_vision_operator.py b/airflow/contrib/operators/gcp_vision_operator.py index 439b72dc539ce..18424cf0bf6fb 100644 --- a/airflow/contrib/operators/gcp_vision_operator.py +++ b/airflow/contrib/operators/gcp_vision_operator.py @@ -56,7 +56,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.vision.CloudVisionImageAnnotateOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -72,7 +72,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.vision.CloudVisionTextDetectOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -88,7 +88,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -104,7 +104,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -120,7 +120,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.vision.CloudVisionGetProductOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -137,7 +137,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductSetOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -154,7 +154,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductSetOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -171,7 +171,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.vision.CloudVisionGetProductSetOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -188,7 +188,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.vision.CloudVisionUpdateProductSetOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -205,7 +205,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.vision.CloudVisionUpdateProductOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -222,6 +222,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.vision.CloudVisionCreateReferenceImageOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcs_acl_operator.py b/airflow/contrib/operators/gcs_acl_operator.py index 80812960c2767..f3e6afa303602 100644 --- a/airflow/contrib/operators/gcs_acl_operator.py +++ b/airflow/contrib/operators/gcs_acl_operator.py @@ -42,7 +42,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.gcs.GCSBucketCreateAclEntryOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -58,6 +58,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.gcs.GCSObjectCreateAclEntryOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcs_delete_operator.py b/airflow/contrib/operators/gcs_delete_operator.py index ede83705f26f2..42fa2a89057d1 100644 --- a/airflow/contrib/operators/gcs_delete_operator.py +++ b/airflow/contrib/operators/gcs_delete_operator.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.gcs.GCSDeleteObjectsOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcs_download_operator.py b/airflow/contrib/operators/gcs_download_operator.py index e36b0b2178b22..a6904320bd2c2 100644 --- a/airflow/contrib/operators/gcs_download_operator.py +++ b/airflow/contrib/operators/gcs_download_operator.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.gcs.GCSToLocalFilesystemOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcs_list_operator.py b/airflow/contrib/operators/gcs_list_operator.py index 3ab582516edeb..a18ec272a5907 100644 --- a/airflow/contrib/operators/gcs_list_operator.py +++ b/airflow/contrib/operators/gcs_list_operator.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.gcs.GCSListObjectsOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcs_operator.py b/airflow/contrib/operators/gcs_operator.py index d394d0e01e56d..4f0aecbf9c167 100644 --- a/airflow/contrib/operators/gcs_operator.py +++ b/airflow/contrib/operators/gcs_operator.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.gcs.GCSCreateBucketOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcs_to_bq.py b/airflow/contrib/operators/gcs_to_bq.py index 525d99a0c315a..dd67be7a06c98 100644 --- a/airflow/contrib/operators/gcs_to_bq.py +++ b/airflow/contrib/operators/gcs_to_bq.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.transfers.gcs_to_bq.GCSToBigQueryOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcs_to_gcs.py b/airflow/contrib/operators/gcs_to_gcs.py index 924991d7ab3b8..ab6f2d91c1ba9 100644 --- a/airflow/contrib/operators/gcs_to_gcs.py +++ b/airflow/contrib/operators/gcs_to_gcs.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSToGCSOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/gcs_to_s3.py b/airflow/contrib/operators/gcs_to_s3.py index b3be699783461..13aa005ab20a3 100644 --- a/airflow/contrib/operators/gcs_to_s3.py +++ b/airflow/contrib/operators/gcs_to_s3.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): "This class is deprecated. " "Please use `airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSToS3Operator`.", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/mlengine_operator.py b/airflow/contrib/operators/mlengine_operator.py index 637c61645fb8a..d5d6b2fb1d24a 100644 --- a/airflow/contrib/operators/mlengine_operator.py +++ b/airflow/contrib/operators/mlengine_operator.py @@ -45,7 +45,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.mlengine.MLEngineStartBatchPredictionJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -61,7 +61,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.mlengine.MLEngineManageModelOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -78,7 +78,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.mlengine.MLEngineStartTrainingJobOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -95,6 +95,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.operators.mlengine.MLEngineManageVersionOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/mssql_to_gcs.py b/airflow/contrib/operators/mssql_to_gcs.py index c14b0ac0a42c2..140457bb709f4 100644 --- a/airflow/contrib/operators/mssql_to_gcs.py +++ b/airflow/contrib/operators/mssql_to_gcs.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.transfers.mssql_to_gcs.MSSQLToGCSOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/mysql_to_gcs.py b/airflow/contrib/operators/mysql_to_gcs.py index d6e491a508fb0..50363b3a39bc6 100644 --- a/airflow/contrib/operators/mysql_to_gcs.py +++ b/airflow/contrib/operators/mysql_to_gcs.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.transfers.mysql_to_gcs.MySQLToGCSOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/oracle_to_oracle_transfer.py b/airflow/contrib/operators/oracle_to_oracle_transfer.py index 6a9ba82f53b67..905cba5a45c21 100644 --- a/airflow/contrib/operators/oracle_to_oracle_transfer.py +++ b/airflow/contrib/operators/oracle_to_oracle_transfer.py @@ -45,6 +45,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.oracle.transfers.oracle_to_oracle.OracleToOracleOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/postgres_to_gcs_operator.py b/airflow/contrib/operators/postgres_to_gcs_operator.py index f9b1fcaa92943..1ce5f42304ce5 100644 --- a/airflow/contrib/operators/postgres_to_gcs_operator.py +++ b/airflow/contrib/operators/postgres_to_gcs_operator.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.transfers.postgres_to_gcs.PostgresToGCSOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/pubsub_operator.py b/airflow/contrib/operators/pubsub_operator.py index 3acbcced80c09..9d85d32fefed1 100644 --- a/airflow/contrib/operators/pubsub_operator.py +++ b/airflow/contrib/operators/pubsub_operator.py @@ -49,7 +49,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.pubsub.PubSubPublishMessageOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -65,7 +65,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.pubsub.PubSubCreateSubscriptionOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -81,7 +81,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.pubsub.PubSubDeleteSubscriptionOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -97,7 +97,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.pubsub.PubSubCreateTopicOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -113,6 +113,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.operators.pubsub.PubSubDeleteTopicOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/sql_to_gcs.py b/airflow/contrib/operators/sql_to_gcs.py index 347d886d81e7b..13aa869c5d98d 100644 --- a/airflow/contrib/operators/sql_to_gcs.py +++ b/airflow/contrib/operators/sql_to_gcs.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.transfers.sql_to_gcs.BaseSQLToGCSOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/vertica_to_hive.py b/airflow/contrib/operators/vertica_to_hive.py index 67444b003b7ba..49ddea172f374 100644 --- a/airflow/contrib/operators/vertica_to_hive.py +++ b/airflow/contrib/operators/vertica_to_hive.py @@ -44,6 +44,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.apache.hive.transfers.vertica_to_hive.VerticaToHiveOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/operators/vertica_to_mysql.py b/airflow/contrib/operators/vertica_to_mysql.py index 961ca57cb4085..35b0df66dc776 100644 --- a/airflow/contrib/operators/vertica_to_mysql.py +++ b/airflow/contrib/operators/vertica_to_mysql.py @@ -45,6 +45,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.mysql.transfers.vertica_to_mysql.VerticaToMySqlOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/sensors/bigquery_sensor.py b/airflow/contrib/sensors/bigquery_sensor.py index a12b4e89f24b1..d58445e3d174b 100644 --- a/airflow/contrib/sensors/bigquery_sensor.py +++ b/airflow/contrib/sensors/bigquery_sensor.py @@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.sensors.bigquery.BigQueryTableExistenceSensor`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/sensors/gcp_transfer_sensor.py b/airflow/contrib/sensors/gcp_transfer_sensor.py index 752ec252e9b4c..429ddb1a44b05 100644 --- a/airflow/contrib/sensors/gcp_transfer_sensor.py +++ b/airflow/contrib/sensors/gcp_transfer_sensor.py @@ -46,6 +46,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.google.cloud.sensors.transfer.CloudDataTransferServiceJobStatusSensor`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/sensors/gcs_sensor.py b/airflow/contrib/sensors/gcs_sensor.py index 8dfaa095372db..3df15f676c265 100644 --- a/airflow/contrib/sensors/gcs_sensor.py +++ b/airflow/contrib/sensors/gcs_sensor.py @@ -44,7 +44,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.sensors.gcs.GCSObjectExistenceSensor`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.sensors.gcs.GCSObjectUpdateSensor`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -76,7 +76,7 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.sensors.gcs.GCSObjectsWithPrefixExistenceSensor`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -92,6 +92,6 @@ def __init__(self, *args, **kwargs): """This class is deprecated. Please use `airflow.providers.google.cloud.sensors.gcs.GCSUploadSessionCompleteSensor`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/contrib/sensors/hdfs_sensor.py b/airflow/contrib/sensors/hdfs_sensor.py index 8860cd6f1e961..1fa3ce6264d1c 100644 --- a/airflow/contrib/sensors/hdfs_sensor.py +++ b/airflow/contrib/sensors/hdfs_sensor.py @@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.apache.hdfs.sensors.hdfs.HdfsFolderSensor`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) @@ -65,6 +65,6 @@ def __init__(self, *args, **kwargs): Please use `airflow.providers.apache.hdfs.sensors.hdfs.HdfsRegexSensor`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(*args, **kwargs) diff --git a/airflow/decorators/base.py b/airflow/decorators/base.py index 672e6d27608a5..47fb0d24dfe72 100644 --- a/airflow/decorators/base.py +++ b/airflow/decorators/base.py @@ -25,7 +25,6 @@ from airflow.models import BaseOperator from airflow.models.dag import DAG, DagContext from airflow.models.xcom_arg import XComArg -from airflow.utils.decorators import apply_defaults from airflow.utils.task_group import TaskGroup, TaskGroupContext @@ -111,7 +110,6 @@ class DecoratedOperator(BaseOperator): # there are some cases we can't deepcopy the objects (e.g protobuf). shallow_copy_attrs = ('python_callable',) - @apply_defaults def __init__( self, *, diff --git a/airflow/decorators/python.py b/airflow/decorators/python.py index 7b5aca532db92..f089995c3af9f 100644 --- a/airflow/decorators/python.py +++ b/airflow/decorators/python.py @@ -19,7 +19,6 @@ from airflow.decorators.base import DecoratedOperator, task_decorator_factory from airflow.operators.python import PythonOperator -from airflow.utils.decorators import apply_defaults class _PythonDecoratedOperator(DecoratedOperator, PythonOperator): @@ -47,7 +46,6 @@ class _PythonDecoratedOperator(DecoratedOperator, PythonOperator): # there are some cases we can't deepcopy the objects (e.g protobuf). shallow_copy_attrs = ('python_callable',) - @apply_defaults def __init__( self, **kwargs, diff --git a/airflow/decorators/python_virtualenv.py b/airflow/decorators/python_virtualenv.py index 7e3cc11fbef60..386c267f9bde7 100644 --- a/airflow/decorators/python_virtualenv.py +++ b/airflow/decorators/python_virtualenv.py @@ -21,7 +21,6 @@ from airflow.decorators.base import DecoratedOperator, task_decorator_factory from airflow.operators.python import PythonVirtualenvOperator -from airflow.utils.decorators import apply_defaults from airflow.utils.python_virtualenv import remove_task_decorator @@ -50,7 +49,6 @@ class _PythonVirtualenvDecoratedOperator(DecoratedOperator, PythonVirtualenvOper # there are some cases we can't deepcopy the objects (e.g protobuf). shallow_copy_attrs = ('python_callable',) - @apply_defaults def __init__( self, **kwargs, diff --git a/airflow/example_dags/tutorial.py b/airflow/example_dags/tutorial.py index 09d6ca3f1a658..6d0b5224cd739 100644 --- a/airflow/example_dags/tutorial.py +++ b/airflow/example_dags/tutorial.py @@ -89,18 +89,21 @@ # [END basic_task] # [START documentation] - dag.doc_md = __doc__ - t1.doc_md = dedent( """\ #### Task Documentation You can document your task using the attributes `doc_md` (markdown), `doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets rendered in the UI's Task Instance Details page. - ![img](http://montcs.bloomu.edu/~bobmon/Semesters/2012-01/491/import%20soul.png) + """ ) + + dag.doc_md = __doc__ # providing that you have a docstring at the beggining of the DAG + dag.doc_md = """ + This is a documentation placed anywhere + """ # otherwise, type it like this # [END documentation] # [START jinja_template] diff --git a/airflow/example_dags/tutorial_taskflow_api_etl_virtualenv.py b/airflow/example_dags/tutorial_taskflow_api_etl_virtualenv.py index 9bf81f863c0fb..6d00cf662ae9d 100644 --- a/airflow/example_dags/tutorial_taskflow_api_etl_virtualenv.py +++ b/airflow/example_dags/tutorial_taskflow_api_etl_virtualenv.py @@ -38,7 +38,7 @@ # [START instantiate_dag] @dag(default_args=default_args, schedule_interval=None, start_date=days_ago(2), tags=['example']) -def tutorial_taskflow_api_etl(): +def tutorial_taskflow_api_etl_virtualenv(): """ ### TaskFlow API Tutorial Documentation This is a simple ETL data pipeline example which demonstrates the use of @@ -107,7 +107,7 @@ def load(total_order_value: float): # [START dag_invocation] -tutorial_etl_dag = tutorial_taskflow_api_etl() +tutorial_etl_dag = tutorial_taskflow_api_etl_virtualenv() # [END dag_invocation] # [END tutorial] diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 183fcc5e4987b..b90467ccd6894 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -98,6 +98,19 @@ class AirflowDagCycleException(AirflowException): """Raise when there is a cycle in Dag definition""" +class AirflowDagDuplicatedIdException(AirflowException): + """Raise when a Dag's ID is already used by another Dag""" + + def __init__(self, dag_id: str, incoming: str, existing: str) -> None: + super().__init__(dag_id, incoming, existing) + self.dag_id = dag_id + self.incoming = incoming + self.existing = existing + + def __str__(self) -> str: + return f"Ignoring DAG {self.dag_id} from {self.incoming} - also found in {self.existing}" + + class AirflowClusterPolicyViolation(AirflowException): """Raise when there is a violation of a Cluster Policy in Dag definition""" diff --git a/airflow/hooks/base.py b/airflow/hooks/base.py index dee76dc70a90f..f286a6f2ce3d3 100644 --- a/airflow/hooks/base.py +++ b/airflow/hooks/base.py @@ -74,8 +74,8 @@ def get_connection(cls, conn_id: str) -> "Connection": conn.port, conn.schema, conn.login, - "XXXXXXXX" if conn.password else None, - "XXXXXXXX" if conn.extra_dejson else None, + conn.password, + conn.extra_dejson, ) return conn diff --git a/airflow/jobs/local_task_job.py b/airflow/jobs/local_task_job.py index b642310614249..9e68450cf2700 100644 --- a/airflow/jobs/local_task_job.py +++ b/airflow/jobs/local_task_job.py @@ -17,7 +17,6 @@ # under the License. # -import os import signal from typing import Optional @@ -152,8 +151,12 @@ def handle_task_exit(self, return_code: int) -> None: self.log.info("Task exited with return code %s", return_code) self.task_instance.refresh_from_db() # task exited by itself, so we need to check for error file - # incase it failed due to runtime exception/error + # in case it failed due to runtime exception/error error = None + if self.task_instance.state == State.RUNNING: + # This is for a case where the task received a sigkill + # while running + self.task_instance.set_state(State.FAILED) if self.task_instance.state != State.SUCCESS: error = self.task_runner.deserialize_run_error() self.task_instance._run_finished_callback(error=error) # pylint: disable=protected-access @@ -184,9 +187,9 @@ def heartbeat_callback(self, session=None): ) raise AirflowException("Hostname of job runner does not match") - current_pid = os.getpid() + current_pid = self.task_runner.process.pid same_process = ti.pid == current_pid - if not same_process: + if ti.pid is not None and not same_process: self.log.warning("Recorded pid %s does not match " "the current pid %s", ti.pid, current_pid) raise AirflowException("PID of job runner does not match") elif self.task_runner.return_code() is None and hasattr(self.task_runner, 'process'): diff --git a/airflow/kubernetes/pod_generator.py b/airflow/kubernetes/pod_generator.py index 9c534e87b930d..80602e3cb9a8e 100644 --- a/airflow/kubernetes/pod_generator.py +++ b/airflow/kubernetes/pod_generator.py @@ -451,7 +451,7 @@ def make_unique_pod_id(pod_id: str) -> str: return None safe_uuid = uuid.uuid4().hex # safe uuid will always be less than 63 chars - # Strip trailing '-' and '.' as they cant be followed by '.' + # Strip trailing '-' and '.' as they can't be followed by '.' trimmed_pod_id = pod_id[:MAX_LABEL_LEN].rstrip('-.') safe_pod_id = f"{trimmed_pod_id}.{safe_uuid}" diff --git a/airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml b/airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml index 686d1d1302f26..27a7e963d5c2d 100644 --- a/airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml +++ b/airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml @@ -65,6 +65,7 @@ spec: restartPolicy: Never securityContext: runAsUser: 50000 + fsGroup: 50000 nodeSelector: {} affinity: diff --git a/airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml b/airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml index cf2daec691f6d..a290cf5cb25dc 100644 --- a/airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml +++ b/airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml @@ -62,6 +62,7 @@ spec: restartPolicy: Never securityContext: runAsUser: 50000 + fsGroup: 50000 nodeSelector: {} affinity: diff --git a/airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml b/airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml index dee6a0861c0cc..445f509139fd2 100644 --- a/airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml +++ b/airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml @@ -31,7 +31,7 @@ spec: - name: GIT_SYNC_REV value: "HEAD" - name: GIT_SYNC_BRANCH - value: "v1-10-stable" + value: "v2-0-stable" - name: GIT_SYNC_REPO value: "https://github.com/apache/airflow.git" - name: GIT_SYNC_DEPTH @@ -86,6 +86,7 @@ spec: restartPolicy: Never securityContext: runAsUser: 50000 + fsGroup: 50000 nodeSelector: {} affinity: diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index ae7a276650e3c..328e50dd8c7ee 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -24,6 +24,7 @@ import warnings from abc import ABCMeta, abstractmethod from datetime import datetime, timedelta +from inspect import signature from typing import ( TYPE_CHECKING, Any, @@ -38,7 +39,9 @@ Set, Tuple, Type, + TypeVar, Union, + cast, ) import attr @@ -65,7 +68,6 @@ from airflow.ti_deps.deps.prev_dagrun_dep import PrevDagrunDep from airflow.ti_deps.deps.trigger_rule_dep import TriggerRuleDep from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults from airflow.utils.edgemodifier import EdgeModifier from airflow.utils.helpers import validate_key from airflow.utils.log.logging_mixin import LoggingMixin @@ -81,23 +83,113 @@ TaskStateChangeCallback = Callable[[Context], None] +T = TypeVar('T', bound=Callable) # pylint: disable=invalid-name + class BaseOperatorMeta(abc.ABCMeta): - """Base metaclass of BaseOperator.""" + """Metaclass of BaseOperator.""" - def __call__(cls, *args, **kwargs): + @classmethod + def _apply_defaults(cls, func: T) -> T: """ - Called when you call BaseOperator(). In this way we are able to perform an action - after initializing an operator no matter where the ``super().__init__`` is called - (before or after assign of new attributes in a custom operator). + Function decorator that Looks for an argument named "default_args", and + fills the unspecified arguments from it. + + Since python2.* isn't clear about which arguments are missing when + calling a function, and that this can be quite confusing with multi-level + inheritance and argument defaults, this decorator also alerts with + specific information about the missing arguments. """ - obj: BaseOperator = type.__call__(cls, *args, **kwargs) - # Here we set upstream task defined by XComArgs passed to template fields of the operator - obj.set_xcomargs_dependencies() + # Cache inspect.signature for the wrapper closure to avoid calling it + # at every decorated invocation. This is separate sig_cache created + # per decoration, i.e. each function decorated using apply_defaults will + # have a different sig_cache. + sig_cache = signature(func) + non_optional_args = { + name + for (name, param) in sig_cache.parameters.items() + if param.default == param.empty + and param.name != 'self' + and param.kind not in (param.VAR_POSITIONAL, param.VAR_KEYWORD) + } + + # pylint: disable=invalid-name,missing-docstring + class autostacklevel_warn: + def __init__(self): + self.warnings = __import__('warnings') + + def __getattr__(self, name): + return getattr(self.warnings, name) + + def __dir__(self): + return dir(self.warnings) + + def warn(self, message, category=None, stacklevel=1, source=None): + self.warnings.warn(message, category, stacklevel + 2, source) + + # pylint: enable=invalid-name,missing-docstring + + if func.__globals__.get('warnings') is sys.modules['warnings']: + # Yes, this is slightly hacky, but it _automatically_ sets the right + # stacklevel parameter to `warnings.warn` to ignore the decorator. Now + # that the decorator is applied automatically, this makes the needed + # stacklevel parameter less confusing. + func.__globals__['warnings'] = autostacklevel_warn() + + @functools.wraps(func) + def apply_defaults(self, *args: Any, **kwargs: Any) -> Any: + from airflow.models.dag import DagContext + + if len(args) > 0: + raise AirflowException("Use keyword arguments when initializing operators") + dag_args: Dict[str, Any] = {} + dag_params: Dict[str, Any] = {} + + dag = kwargs.get('dag') or DagContext.get_current_dag() + if dag: + dag_args = copy.copy(dag.default_args) or {} + dag_params = copy.copy(dag.params) or {} + + params = kwargs.get('params', {}) or {} + dag_params.update(params) + + default_args = {} + if 'default_args' in kwargs: + default_args = kwargs['default_args'] + if 'params' in default_args: + dag_params.update(default_args['params']) + del default_args['params'] + + dag_args.update(default_args) + default_args = dag_args + + for arg in sig_cache.parameters: + if arg not in kwargs and arg in default_args: + kwargs[arg] = default_args[arg] + + missing_args = list(non_optional_args - set(kwargs)) + if missing_args: + msg = f"Argument {missing_args} is required" + raise AirflowException(msg) + + if dag_params: + kwargs['params'] = dag_params + + result = func(self, *args, **kwargs) + + # Here we set upstream task defined by XComArgs passed to template fields of the operator + self.set_xcomargs_dependencies() + + # Mark instance as instantiated https://docs.python.org/3/tutorial/classes.html#private-variables + self._BaseOperator__instantiated = True # pylint: disable=protected-access + return result + + return cast(T, apply_defaults) - # Mark instance as instantiated https://docs.python.org/3/tutorial/classes.html#private-variables - obj._BaseOperator__instantiated = True - return obj + def __new__(cls, name, bases, namespace): + new_cls = super().__new__(cls, name, bases, namespace) + new_cls.__init__ = cls._apply_defaults(new_cls.__init__) + return new_cls # pylint: disable=too-many-instance-attributes,too-many-public-methods @@ -362,7 +454,6 @@ class derived from this one results in the creation of a task object, _lock_for_execution = False # pylint: disable=too-many-arguments,too-many-locals, too-many-statements - @apply_defaults def __init__( self, task_id: str, diff --git a/airflow/models/connection.py b/airflow/models/connection.py index f631aaef2c677..9021edb56a9e9 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -24,7 +24,7 @@ from sqlalchemy import Boolean, Column, Integer, String, Text from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm import synonym +from sqlalchemy.orm import reconstructor, synonym from airflow.configuration import ensure_secrets_loaded from airflow.exceptions import AirflowException, AirflowNotFoundException @@ -32,6 +32,7 @@ from airflow.models.crypto import get_fernet from airflow.providers_manager import ProvidersManager from airflow.utils.log.logging_mixin import LoggingMixin +from airflow.utils.log.secrets_masker import mask_secret from airflow.utils.module_loading import import_string @@ -141,6 +142,14 @@ def __init__( # pylint: disable=too-many-arguments self.port = port self.extra = extra + if self.password: + mask_secret(self.password) + + @reconstructor + def on_db_load(self): # pylint: disable=missing-function-docstring + if self.password: + mask_secret(self.password) + def parse_from_uri(self, **uri): """This method is deprecated. Please use uri parameter in constructor.""" warnings.warn( @@ -346,9 +355,13 @@ def extra_dejson(self) -> Dict: if self.extra: try: obj = json.loads(self.extra) + except JSONDecodeError: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) + # Mask sensitive keys from this list + mask_secret(obj) + return obj @classmethod diff --git a/airflow/models/dag.py b/airflow/models/dag.py index b55698eb2742f..861d33f359260 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -48,6 +48,7 @@ import pendulum from croniter import croniter from dateutil.relativedelta import relativedelta +from jinja2.nativetypes import NativeEnvironment from sqlalchemy import Boolean, Column, ForeignKey, Index, Integer, String, Text, func, or_ from sqlalchemy.orm import backref, joinedload, relationship from sqlalchemy.orm.session import Session @@ -65,6 +66,7 @@ from airflow.models.taskinstance import Context, TaskInstance, clear_task_instances from airflow.security import permissions from airflow.stats import Stats +from airflow.typing_compat import RePatternType from airflow.utils import timezone from airflow.utils.dates import cron_presets, date_range as utils_date_range from airflow.utils.file import correct_maybe_zipped @@ -79,13 +81,6 @@ from airflow.utils.task_group import TaskGroup -# Before Py 3.7, there is no re.Pattern class -try: - from re import Pattern as PatternType # type: ignore -except ImportError: - PatternType = type(re.compile('', 0)) - - log = logging.getLogger(__name__) ScheduleInterval = Union[str, timedelta, relativedelta] @@ -263,6 +258,7 @@ def __init__( access_control: Optional[Dict] = None, is_paused_upon_creation: Optional[bool] = None, jinja_environment_kwargs: Optional[Dict] = None, + render_template_as_native_obj: bool = False, tags: Optional[List[str]] = None, ): from airflow.utils.task_group import TaskGroup @@ -365,6 +361,7 @@ def __init__( self.is_paused_upon_creation = is_paused_upon_creation self.jinja_environment_kwargs = jinja_environment_kwargs + self.render_template_as_native_obj = render_template_as_native_obj self.tags = tags self._task_group = TaskGroup.create_root(self) @@ -997,8 +994,10 @@ def get_template_env(self) -> jinja2.Environment: } if self.jinja_environment_kwargs: jinja_env_options.update(self.jinja_environment_kwargs) - - env = jinja2.Environment(**jinja_env_options) # type: ignore + if self.render_template_as_native_obj: + env = NativeEnvironment(**jinja_env_options) + else: + env = jinja2.Environment(**jinja_env_options) # type: ignore # Add any user defined items. Safe to edit globals as long as no templates are rendered yet. # http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment.globals @@ -1444,7 +1443,7 @@ def sub_dag(self, *args, **kwargs): def partial_subset( self, - task_ids_or_regex: Union[str, PatternType, Iterable[str]], + task_ids_or_regex: Union[str, RePatternType, Iterable[str]], include_downstream=False, include_upstream=True, include_direct_upstream=False, @@ -1472,7 +1471,7 @@ def partial_subset( self.task_dict = task_dict self._task_group = task_group - if isinstance(task_ids_or_regex, (str, PatternType)): + if isinstance(task_ids_or_regex, (str, RePatternType)): matched_tasks = [t for t in self.tasks if re.findall(task_ids_or_regex, t.task_id)] else: matched_tasks = [t for t in self.tasks if t.task_id in task_ids_or_regex] diff --git a/airflow/models/dagbag.py b/airflow/models/dagbag.py index 12113e3f2f3df..b78463b6cf5e6 100644 --- a/airflow/models/dagbag.py +++ b/airflow/models/dagbag.py @@ -36,7 +36,12 @@ from airflow import settings from airflow.configuration import conf -from airflow.exceptions import AirflowClusterPolicyViolation, AirflowDagCycleException, SerializedDagNotFound +from airflow.exceptions import ( + AirflowClusterPolicyViolation, + AirflowDagCycleException, + AirflowDagDuplicatedIdException, + SerializedDagNotFound, +) from airflow.stats import Stats from airflow.utils import timezone from airflow.utils.dag_cycle_tester import test_cycle @@ -215,8 +220,15 @@ def get_dag(self, dag_id, session: Session = None): # If the dag corresponding to root_dag_id is absent or expired is_missing = root_dag_id not in self.dags is_expired = orm_dag.last_expired and dag and dag.last_loaded < orm_dag.last_expired + if is_expired: + # Remove associated dags so we can re-add them. + self.dags = { + key: dag + for key, dag in self.dags.items() + if root_dag_id != key and not (dag.is_subdag and root_dag_id == dag.parent_dag.dag_id) + } if is_missing or is_expired: - # Reprocess source file + # Reprocess source file. found_dags = self.process_file( filepath=correct_maybe_zipped(orm_dag.fileloc), only_if_updated=False ) @@ -381,7 +393,11 @@ def _process_modules(self, filepath, mods, file_last_changed_on_disk): self.log.exception("Failed to bag_dag: %s", dag.full_filepath) self.import_errors[dag.full_filepath] = f"Invalid Cron expression: {cron_e}" self.file_last_changed[dag.full_filepath] = file_last_changed_on_disk - except (AirflowDagCycleException, AirflowClusterPolicyViolation) as exception: + except ( + AirflowDagCycleException, + AirflowDagDuplicatedIdException, + AirflowClusterPolicyViolation, + ) as exception: self.log.exception("Failed to bag_dag: %s", dag.full_filepath) self.import_errors[dag.full_filepath] = str(exception) self.file_last_changed[dag.full_filepath] = file_last_changed_on_disk @@ -390,7 +406,17 @@ def _process_modules(self, filepath, mods, file_last_changed_on_disk): def bag_dag(self, dag, root_dag): """ Adds the DAG into the bag, recurses into sub dags. - Throws AirflowDagCycleException if a cycle is detected in this dag or its subdags + + :raises: AirflowDagCycleException if a cycle is detected in this dag or its subdags. + :raises: AirflowDagDuplicatedIdException if this dag or its subdags already exists in the bag. + """ + self._bag_dag(dag=dag, root_dag=root_dag, recursive=True) + + def _bag_dag(self, *, dag, root_dag, recursive): + """Actual implementation of bagging a dag. + + The only purpose of this is to avoid exposing ``recursive`` in ``bag_dag()``, + intended to only be used by the ``_bag_dag()`` implementation. """ test_cycle(dag) # throws if a task cycle is found @@ -406,24 +432,34 @@ def bag_dag(self, dag, root_dag): subdags = dag.subdags try: - for subdag in subdags: - subdag.full_filepath = dag.full_filepath - subdag.parent_dag = dag - subdag.is_subdag = True - self.bag_dag(dag=subdag, root_dag=root_dag) - + # DAG.subdags automatically performs DFS search, so we don't recurse + # into further _bag_dag() calls. + if recursive: + for subdag in subdags: + subdag.full_filepath = dag.full_filepath + subdag.parent_dag = dag + subdag.is_subdag = True + self._bag_dag(dag=subdag, root_dag=root_dag, recursive=False) + + prev_dag = self.dags.get(dag.dag_id) + if prev_dag and prev_dag.full_filepath != dag.full_filepath: + raise AirflowDagDuplicatedIdException( + dag_id=dag.dag_id, + incoming=dag.full_filepath, + existing=self.dags[dag.dag_id].full_filepath, + ) self.dags[dag.dag_id] = dag self.log.debug('Loaded DAG %s', dag) - except AirflowDagCycleException as cycle_exception: + except (AirflowDagCycleException, AirflowDagDuplicatedIdException): # There was an error in bagging the dag. Remove it from the list of dags self.log.exception('Exception bagging dag: %s', dag.dag_id) # Only necessary at the root level since DAG.subdags automatically # performs DFS to search through all subdags - if dag == root_dag: + if recursive: for subdag in subdags: if subdag.dag_id in self.dags: del self.dags[subdag.dag_id] - raise cycle_exception + raise def collect_dags( self, @@ -538,7 +574,7 @@ def _serialize_dag_capturing_errors(dag, session): if dag.is_subdag: return [] try: - # We cant use bulk_write_to_db as we want to capture each error individually + # We can't use bulk_write_to_db as we want to capture each error individually dag_was_updated = SerializedDagModel.write_dag( dag, min_update_interval=settings.MIN_SERIALIZED_DAG_UPDATE_INTERVAL, diff --git a/airflow/models/renderedtifields.py b/airflow/models/renderedtifields.py index 91ec0278df9cf..0572c89acbcb3 100644 --- a/airflow/models/renderedtifields.py +++ b/airflow/models/renderedtifields.py @@ -57,9 +57,20 @@ def __init__(self, ti: TaskInstance, render_templates=True): field: serialize_template_field(getattr(self.task, field)) for field in self.task.template_fields } + self._redact() + def __repr__(self): return f"<{self.__class__.__name__}: {self.dag_id}.{self.task_id} {self.execution_date}" + def _redact(self): + from airflow.utils.log.secrets_masker import redact + + if self.k8s_pod_yaml: + self.k8s_pod_yaml = redact(self.k8s_pod_yaml) + + for field, rendered in self.rendered_fields.items(): + self.rendered_fields[field] = redact(rendered, field) + @classmethod @provide_session def get_templated_fields(cls, ti: TaskInstance, session: Session = None) -> Optional[dict]: diff --git a/airflow/models/serialized_dag.py b/airflow/models/serialized_dag.py index c2706f492fab8..81448b6e64b8d 100644 --- a/airflow/models/serialized_dag.py +++ b/airflow/models/serialized_dag.py @@ -27,12 +27,13 @@ from sqlalchemy import BigInteger, Column, Index, String, and_ from sqlalchemy.orm import Session, backref, foreign, relationship from sqlalchemy.sql import exists +from sqlalchemy.sql.expression import func from airflow.models.base import ID_LEN, Base from airflow.models.dag import DAG, DagModel from airflow.models.dagcode import DagCode from airflow.models.dagrun import DagRun -from airflow.serialization.serialized_objects import SerializedDAG +from airflow.serialization.serialized_objects import DagDependency, SerializedDAG from airflow.settings import MIN_SERIALIZED_DAG_UPDATE_INTERVAL, json from airflow.utils import timezone from airflow.utils.session import provide_session @@ -272,6 +273,17 @@ def get_last_updated_datetime(cls, dag_id: str, session: Session = None) -> date """ return session.query(cls.last_updated).filter(cls.dag_id == dag_id).scalar() + @classmethod + @provide_session + def get_max_last_updated_datetime(cls, session: Session = None) -> datetime: + """ + Get the maximum date when any DAG was last updated in serialized_dag table + + :param session: ORM Session + :type session: Session + """ + return session.query(func.max(cls.last_updated)).scalar() + @classmethod @provide_session def get_latest_version_hash(cls, dag_id: str, session: Session = None) -> str: @@ -286,3 +298,26 @@ def get_latest_version_hash(cls, dag_id: str, session: Session = None) -> str: :rtype: str """ return session.query(cls.dag_hash).filter(cls.dag_id == dag_id).scalar() + + @classmethod + @provide_session + def get_dag_dependencies(cls, session: Session = None) -> Dict[str, List['DagDependency']]: + """ + Get the dependencies between DAGs + + :param session: ORM Session + :type session: Session + """ + dependencies = {} + + if session.bind.dialect.name in ["sqlite", "mysql"]: + for row in session.query(cls.dag_id, func.json_extract(cls.data, "$.dag.dag_dependencies")).all(): + dependencies[row[0]] = [DagDependency(**d) for d in json.loads(row[1])] + + else: + for row in session.query( + cls.dag_id, func.json_extract_path(cls.data, "dag", "dag_dependencies") + ).all(): + dependencies[row[0]] = [DagDependency(**d) for d in row[1]] + + return dependencies diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index d51a5af405a50..a377964609b3b 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -283,7 +283,7 @@ class TaskInstance(Base, LoggingMixin): # pylint: disable=R0902,R0904 external_executor_id = Column(String(ID_LEN, **COLLATION_ARGS)) # If adding new fields here then remember to add them to - # refresh_from_db() or they wont display in the UI correctly + # refresh_from_db() or they won't display in the UI correctly __table_args__ = ( Index('ti_dag_state', dag_id, state), @@ -1072,7 +1072,6 @@ def check_and_change_state_before_execution( # pylint: disable=too-many-argumen if not test_mode: session.add(Log(State.RUNNING, self)) self.state = State.RUNNING - self.pid = os.getpid() self.end_date = None if not test_mode: session.merge(self) @@ -1127,7 +1126,9 @@ def _run_raw_task( self.refresh_from_db(session=session) self.job_id = job_id self.hostname = get_hostname() - + self.pid = os.getpid() + session.merge(self) + session.commit() actual_start_date = timezone.utcnow() Stats.incr(f'ti.start.{task.dag_id}.{task.task_id}') try: @@ -1836,10 +1837,14 @@ def get_email_subject_content(self, exception): max_tries=self.max_tries, ) ) - - jinja_env = jinja2.Environment( - loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), autoescape=True - ) + if self.dag.render_template_as_native_obj: + jinja_env = jinja2.nativetypes.NativeEnvironment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), autoescape=True + ) + else: + jinja_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), autoescape=True + ) subject = jinja_env.from_string(default_subject).render(**jinja_context) html_content = jinja_env.from_string(default_html_content).render(**jinja_context) html_content_err = jinja_env.from_string(default_html_content_err).render(**jinja_context) diff --git a/airflow/models/variable.py b/airflow/models/variable.py index 0341e7b2c65dd..44627c0fdefa2 100644 --- a/airflow/models/variable.py +++ b/airflow/models/variable.py @@ -24,12 +24,13 @@ from cryptography.fernet import InvalidToken as InvalidFernetToken from sqlalchemy import Boolean, Column, Integer, String, Text from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm import Session, synonym +from sqlalchemy.orm import Session, reconstructor, synonym from airflow.configuration import ensure_secrets_loaded from airflow.models.base import ID_LEN, Base from airflow.models.crypto import get_fernet from airflow.utils.log.logging_mixin import LoggingMixin +from airflow.utils.log.secrets_masker import mask_secret from airflow.utils.session import provide_session log = logging.getLogger() @@ -56,6 +57,11 @@ def __init__(self, key=None, val=None, description=None): self.val = val self.description = description + @reconstructor + def on_db_load(self): # pylint: disable=missing-function-docstring + if self._val: + mask_secret(self.val, self.key) + def __repr__(self): # Hiding the value return f'{self.key} : {self._val}' @@ -134,8 +140,11 @@ def get( raise KeyError(f'Variable {key} does not exist') else: if deserialize_json: - return json.loads(var_val) + obj = json.loads(var_val) + mask_secret(var_val, key) + return obj else: + mask_secret(var_val, key) return var_val @classmethod diff --git a/airflow/operators/bash.py b/airflow/operators/bash.py index 4f2409560c953..5f6a68dd16d39 100644 --- a/airflow/operators/bash.py +++ b/airflow/operators/bash.py @@ -26,7 +26,6 @@ from airflow.exceptions import AirflowException, AirflowSkipException from airflow.hooks.subprocess import SubprocessHook from airflow.models import BaseOperator -from airflow.utils.decorators import apply_defaults from airflow.utils.operator_helpers import context_to_airflow_vars @@ -132,7 +131,6 @@ class BashOperator(BaseOperator): ) ui_color = '#f0ede4' - @apply_defaults def __init__( self, *, diff --git a/airflow/operators/check_operator.py b/airflow/operators/check_operator.py index 85da900eb4762..130eb6e577c9c 100644 --- a/airflow/operators/check_operator.py +++ b/airflow/operators/check_operator.py @@ -43,7 +43,7 @@ def __init__(self, **kwargs): """This class is deprecated. Please use `airflow.operators.sql.SQLCheckOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(**kwargs) @@ -59,7 +59,7 @@ def __init__(self, **kwargs): """This class is deprecated. Please use `airflow.operators.sql.SQLIntervalCheckOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(**kwargs) @@ -75,7 +75,7 @@ def __init__(self, **kwargs): """This class is deprecated. Please use `airflow.operators.sql.SQLThresholdCheckOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(**kwargs) @@ -91,6 +91,6 @@ def __init__(self, **kwargs): """This class is deprecated. Please use `airflow.operators.sql.SQLValueCheckOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(**kwargs) diff --git a/airflow/operators/datetime.py b/airflow/operators/datetime.py index 63aea910a8f0f..6b1acf72b4b41 100644 --- a/airflow/operators/datetime.py +++ b/airflow/operators/datetime.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.operators.branch import BaseBranchOperator from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults class BranchDateTimeOperator(BaseBranchOperator): @@ -48,7 +47,6 @@ class BranchDateTimeOperator(BaseBranchOperator): :type use_task_execution_date: bool """ - @apply_defaults def __init__( self, *, diff --git a/airflow/operators/dummy.py b/airflow/operators/dummy.py index 13d27508f4f86..3b77104ea5283 100644 --- a/airflow/operators/dummy.py +++ b/airflow/operators/dummy.py @@ -17,7 +17,6 @@ # under the License. from airflow.models import BaseOperator -from airflow.utils.decorators import apply_defaults class DummyOperator(BaseOperator): @@ -31,7 +30,6 @@ class DummyOperator(BaseOperator): ui_color = '#e8f7e4' inherits_from_dummy_operator = True - @apply_defaults def __init__(self, **kwargs) -> None: super().__init__(**kwargs) diff --git a/airflow/operators/email.py b/airflow/operators/email.py index 5ae5f8022504b..69807f21052c4 100644 --- a/airflow/operators/email.py +++ b/airflow/operators/email.py @@ -18,7 +18,6 @@ from typing import List, Optional, Union from airflow.models import BaseOperator -from airflow.utils.decorators import apply_defaults from airflow.utils.email import send_email @@ -51,7 +50,6 @@ class EmailOperator(BaseOperator): template_ext = ('.html',) ui_color = '#e6faf9' - @apply_defaults def __init__( # pylint: disable=invalid-name self, *, diff --git a/airflow/operators/generic_transfer.py b/airflow/operators/generic_transfer.py index a5242b3ad0cd4..55f6384925713 100644 --- a/airflow/operators/generic_transfer.py +++ b/airflow/operators/generic_transfer.py @@ -19,7 +19,6 @@ from airflow.hooks.base import BaseHook from airflow.models import BaseOperator -from airflow.utils.decorators import apply_defaults class GenericTransfer(BaseOperator): @@ -52,7 +51,6 @@ class GenericTransfer(BaseOperator): template_fields_renderers = {"preoperator": "sql"} ui_color = '#b0f07c' - @apply_defaults def __init__( self, *, diff --git a/airflow/operators/presto_check_operator.py b/airflow/operators/presto_check_operator.py index 197bcb6ccde7c..b5731626c541d 100644 --- a/airflow/operators/presto_check_operator.py +++ b/airflow/operators/presto_check_operator.py @@ -38,7 +38,7 @@ def __init__(self, **kwargs): """This class is deprecated. Please use `airflow.operators.sql.SQLCheckOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(**kwargs) @@ -56,7 +56,7 @@ def __init__(self, **kwargs): Please use `airflow.operators.sql.SQLIntervalCheckOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(**kwargs) @@ -74,6 +74,6 @@ def __init__(self, **kwargs): Please use `airflow.operators.sql.SQLValueCheckOperator`. """, DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(**kwargs) diff --git a/airflow/operators/python.py b/airflow/operators/python.py index c19f89834b91f..e43425aa504f3 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -31,7 +31,6 @@ from airflow.models import BaseOperator from airflow.models.skipmixin import SkipMixin from airflow.models.taskinstance import _CURRENT_CONTEXT -from airflow.utils.decorators import apply_defaults from airflow.utils.operator_helpers import determine_kwargs from airflow.utils.process_utils import execute_in_subprocess from airflow.utils.python_virtualenv import prepare_virtualenv, write_python_script @@ -115,7 +114,6 @@ class PythonOperator(BaseOperator): 'op_kwargs', ) - @apply_defaults def __init__( self, *, @@ -297,7 +295,6 @@ class PythonVirtualenvOperator(PythonOperator): } AIRFLOW_SERIALIZABLE_CONTEXT_KEYS = {'macros', 'conf', 'dag', 'dag_run', 'task'} - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -377,6 +374,7 @@ def execute_callable(self): python_callable_source=self.get_python_source(), ), filename=script_filename, + render_template_as_native_obj=self.dag.render_template_as_native_obj, ) execute_in_subprocess( diff --git a/airflow/operators/sql.py b/airflow/operators/sql.py index 7c7851ced379f..092adc71e0a17 100644 --- a/airflow/operators/sql.py +++ b/airflow/operators/sql.py @@ -27,7 +27,6 @@ from airflow.hooks.base import BaseHook from airflow.hooks.dbapi import DbApiHook from airflow.models import BaseOperator, SkipMixin -from airflow.utils.decorators import apply_defaults class BaseSQLOperator(BaseOperator): @@ -39,7 +38,6 @@ class BaseSQLOperator(BaseOperator): You can custom the behavior by overriding the .get_db_hook() method. """ - @apply_defaults def __init__(self, *, conn_id: Optional[str] = None, database: Optional[str] = None, **kwargs): super().__init__(**kwargs) self.conn_id = conn_id @@ -116,7 +114,6 @@ class SQLCheckOperator(BaseSQLOperator): ) ui_color = "#fff7e6" - @apply_defaults def __init__( self, *, sql: str, conn_id: Optional[str] = None, database: Optional[str] = None, **kwargs ) -> None: @@ -174,7 +171,6 @@ class SQLValueCheckOperator(BaseSQLOperator): ) # type: Iterable[str] ui_color = "#fff7e6" - @apply_defaults def __init__( self, *, @@ -289,7 +285,6 @@ class SQLIntervalCheckOperator(BaseSQLOperator): "relative_diff": lambda cur, ref: float(abs(cur - ref)) / ref, } - @apply_defaults def __init__( self, *, @@ -412,7 +407,6 @@ class SQLThresholdCheckOperator(BaseSQLOperator): ".sql", ) # type: Iterable[str] - @apply_defaults def __init__( self, *, @@ -499,7 +493,6 @@ class BranchSQLOperator(BaseSQLOperator, SkipMixin): ui_color = "#a22034" ui_fgcolor = "#F7F7F7" - @apply_defaults def __init__( self, *, diff --git a/airflow/operators/sql_branch_operator.py b/airflow/operators/sql_branch_operator.py index abb47c94fa47b..5987bce61003f 100644 --- a/airflow/operators/sql_branch_operator.py +++ b/airflow/operators/sql_branch_operator.py @@ -35,6 +35,6 @@ def __init__(self, **kwargs): """This class is deprecated. Please use `airflow.operators.sql.BranchSQLOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(**kwargs) diff --git a/airflow/operators/subdag.py b/airflow/operators/subdag.py index 67598d98aa528..300d1e80255cf 100644 --- a/airflow/operators/subdag.py +++ b/airflow/operators/subdag.py @@ -28,7 +28,6 @@ from airflow.models.pool import Pool from airflow.models.taskinstance import TaskInstance from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults from airflow.utils.session import create_session, provide_session from airflow.utils.state import State from airflow.utils.types import DagRunType @@ -62,7 +61,6 @@ class SubDagOperator(BaseSensorOperator): ui_fgcolor = '#fff' @provide_session - @apply_defaults def __init__( self, *, diff --git a/airflow/operators/trigger_dagrun.py b/airflow/operators/trigger_dagrun.py index 26202dbf1b37a..8f82a7ebab926 100644 --- a/airflow/operators/trigger_dagrun.py +++ b/airflow/operators/trigger_dagrun.py @@ -25,7 +25,6 @@ from airflow.exceptions import AirflowException, DagNotFound, DagRunAlreadyExists from airflow.models import BaseOperator, BaseOperatorLink, DagBag, DagModel, DagRun from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults from airflow.utils.helpers import build_airflow_url_with_query from airflow.utils.state import State from airflow.utils.types import DagRunType @@ -79,7 +78,6 @@ def operator_extra_links(self): """Return operator extra links""" return [TriggerDagRunLink()] - @apply_defaults def __init__( self, *, diff --git a/airflow/operators/weekday.py b/airflow/operators/weekday.py index 2f1a3670d426b..ebb5a2ded714f 100644 --- a/airflow/operators/weekday.py +++ b/airflow/operators/weekday.py @@ -20,7 +20,6 @@ from airflow.operators.branch import BaseBranchOperator from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults from airflow.utils.weekday import WeekDay @@ -50,7 +49,6 @@ class BranchDayOfWeekOperator(BaseBranchOperator): :type use_task_execution_day: bool """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/airbyte/hooks/airbyte.py b/airflow/providers/airbyte/hooks/airbyte.py index 0aeb4f887f321..81d4a0523c1d5 100644 --- a/airflow/providers/airbyte/hooks/airbyte.py +++ b/airflow/providers/airbyte/hooks/airbyte.py @@ -87,7 +87,7 @@ def submit_sync_connection(self, connection_id: str) -> Any: Submits a job to a Airbyte server. :param connection_id: Required. The ConnectionId of the Airbyte Connection. - :type connectiond_id: str + :type connection_id: str """ return self.run( endpoint=f"api/{self.api_version}/connections/sync", diff --git a/airflow/providers/airbyte/operators/airbyte.py b/airflow/providers/airbyte/operators/airbyte.py index 6932fa31a2a9d..aac4563eb4e2c 100644 --- a/airflow/providers/airbyte/operators/airbyte.py +++ b/airflow/providers/airbyte/operators/airbyte.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.airbyte.hooks.airbyte import AirbyteHook -from airflow.utils.decorators import apply_defaults class AirbyteTriggerSyncOperator(BaseOperator): @@ -51,7 +50,6 @@ class AirbyteTriggerSyncOperator(BaseOperator): template_fields = ('connection_id',) - @apply_defaults def __init__( self, connection_id: str, diff --git a/airflow/providers/airbyte/sensors/airbyte.py b/airflow/providers/airbyte/sensors/airbyte.py index 9799ade881f2e..dc202d0f3c9af 100644 --- a/airflow/providers/airbyte/sensors/airbyte.py +++ b/airflow/providers/airbyte/sensors/airbyte.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.providers.airbyte.hooks.airbyte import AirbyteHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class AirbyteJobSensor(BaseSensorOperator): @@ -40,7 +39,6 @@ class AirbyteJobSensor(BaseSensorOperator): template_fields = ('airbyte_job_id',) ui_color = '#6C51FD' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/example_dags/example_glacier_to_gcs.py b/airflow/providers/amazon/aws/example_dags/example_glacier_to_gcs.py index 87d6aa6221c69..a868b0a55bc7c 100644 --- a/airflow/providers/amazon/aws/example_dags/example_glacier_to_gcs.py +++ b/airflow/providers/amazon/aws/example_dags/example_glacier_to_gcs.py @@ -23,7 +23,7 @@ from airflow.utils.dates import days_ago VAULT_NAME = "airflow" -BUCKET_NAME = os.environ.get("GLACIER_GCS_BUCKET_NAME", "gs://glacier_bucket") +BUCKET_NAME = os.environ.get("GLACIER_GCS_BUCKET_NAME", "gs://INVALID BUCKET NAME") OBJECT_NAME = os.environ.get("GLACIER_OBJECT", "example-text.txt") with models.DAG( diff --git a/airflow/providers/amazon/aws/hooks/glue.py b/airflow/providers/amazon/aws/hooks/glue.py index 305acaf98eb7d..8d1cba28bcf50 100644 --- a/airflow/providers/amazon/aws/hooks/glue.py +++ b/airflow/providers/amazon/aws/hooks/glue.py @@ -168,7 +168,7 @@ def get_or_create_glue_job(self) -> str: return get_job_response['Job']['Name'] except glue_client.exceptions.EntityNotFoundException: - self.log.info("Job doesnt exist. Now creating and running AWS Glue Job") + self.log.info("Job doesn't exist. Now creating and running AWS Glue Job") if self.s3_bucket is None: raise AirflowException('Could not initialize glue job, error: Specify Parameter `s3_bucket`') s3_log_path = f's3://{self.s3_bucket}/{self.s3_glue_logs}{self.job_name}' diff --git a/airflow/providers/amazon/aws/operators/athena.py b/airflow/providers/amazon/aws/operators/athena.py index 961fc8a1385f7..0b7d77d8e4b6b 100644 --- a/airflow/providers/amazon/aws/operators/athena.py +++ b/airflow/providers/amazon/aws/operators/athena.py @@ -26,7 +26,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.athena import AWSAthenaHook -from airflow.utils.decorators import apply_defaults class AWSAthenaOperator(BaseOperator): @@ -60,7 +59,6 @@ class AWSAthenaOperator(BaseOperator): template_ext = ('.sql',) template_fields_renderers = {"query": "sql"} - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/amazon/aws/operators/batch.py b/airflow/providers/amazon/aws/operators/batch.py index 1d3e453846643..296b81e41cb81 100644 --- a/airflow/providers/amazon/aws/operators/batch.py +++ b/airflow/providers/amazon/aws/operators/batch.py @@ -31,7 +31,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.batch_client import AwsBatchClientHook -from airflow.utils.decorators import apply_defaults class AwsBatchOperator(BaseOperator): @@ -102,7 +101,6 @@ class AwsBatchOperator(BaseOperator): ) template_fields_renderers = {"overrides": "py", "parameters": "py"} - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/cloud_formation.py b/airflow/providers/amazon/aws/operators/cloud_formation.py index 0f511c30bcc21..e247ca5968f03 100644 --- a/airflow/providers/amazon/aws/operators/cloud_formation.py +++ b/airflow/providers/amazon/aws/operators/cloud_formation.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.cloud_formation import AWSCloudFormationHook -from airflow.utils.decorators import apply_defaults class CloudFormationCreateStackOperator(BaseOperator): @@ -42,7 +41,6 @@ class CloudFormationCreateStackOperator(BaseOperator): template_ext = () ui_color = '#6b9659' - @apply_defaults def __init__(self, *, stack_name: str, params: dict, aws_conn_id: str = 'aws_default', **kwargs): super().__init__(**kwargs) self.stack_name = stack_name @@ -76,7 +74,6 @@ class CloudFormationDeleteStackOperator(BaseOperator): ui_color = '#1d472b' ui_fgcolor = '#FFF' - @apply_defaults def __init__( self, *, stack_name: str, params: Optional[dict] = None, aws_conn_id: str = 'aws_default', **kwargs ): diff --git a/airflow/providers/amazon/aws/operators/datasync.py b/airflow/providers/amazon/aws/operators/datasync.py index e312a46a8d18b..6c88eb1c339cd 100644 --- a/airflow/providers/amazon/aws/operators/datasync.py +++ b/airflow/providers/amazon/aws/operators/datasync.py @@ -24,7 +24,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.datasync import AWSDataSyncHook -from airflow.utils.decorators import apply_defaults # pylint: disable=too-many-instance-attributes, too-many-arguments @@ -125,7 +124,6 @@ class AWSDataSyncOperator(BaseOperator): } ui_color = "#44b5e2" - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/ec2_start_instance.py b/airflow/providers/amazon/aws/operators/ec2_start_instance.py index 157fdd8445575..f6e324146d66a 100644 --- a/airflow/providers/amazon/aws/operators/ec2_start_instance.py +++ b/airflow/providers/amazon/aws/operators/ec2_start_instance.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.ec2 import EC2Hook -from airflow.utils.decorators import apply_defaults class EC2StartInstanceOperator(BaseOperator): @@ -43,7 +42,6 @@ class EC2StartInstanceOperator(BaseOperator): ui_color = "#eeaa11" ui_fgcolor = "#ffffff" - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/ec2_stop_instance.py b/airflow/providers/amazon/aws/operators/ec2_stop_instance.py index fba4652b06574..e32090739a466 100644 --- a/airflow/providers/amazon/aws/operators/ec2_stop_instance.py +++ b/airflow/providers/amazon/aws/operators/ec2_stop_instance.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.ec2 import EC2Hook -from airflow.utils.decorators import apply_defaults class EC2StopInstanceOperator(BaseOperator): @@ -43,7 +42,6 @@ class EC2StopInstanceOperator(BaseOperator): ui_color = "#eeaa11" ui_fgcolor = "#ffffff" - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index cf834ee0bda1c..01b130748bfa8 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -29,7 +29,6 @@ from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook from airflow.typing_compat import Protocol, runtime_checkable -from airflow.utils.decorators import apply_defaults def should_retry(exception: Exception): @@ -145,7 +144,6 @@ class ECSOperator(BaseOperator): # pylint: disable=too-many-instance-attributes template_fields = ('overrides',) template_fields_renderers = {"overrides": "py"} - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/emr_add_steps.py b/airflow/providers/amazon/aws/operators/emr_add_steps.py index 2ffd5cce8d3e3..a410aa36fa303 100644 --- a/airflow/providers/amazon/aws/operators/emr_add_steps.py +++ b/airflow/providers/amazon/aws/operators/emr_add_steps.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.emr import EmrHook -from airflow.utils.decorators import apply_defaults class EmrAddStepsOperator(BaseOperator): @@ -50,7 +49,6 @@ class EmrAddStepsOperator(BaseOperator): template_ext = ('.json',) ui_color = '#f9c915' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/emr_create_job_flow.py b/airflow/providers/amazon/aws/operators/emr_create_job_flow.py index ccd3d544749d8..0b8aace12e442 100644 --- a/airflow/providers/amazon/aws/operators/emr_create_job_flow.py +++ b/airflow/providers/amazon/aws/operators/emr_create_job_flow.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.emr import EmrHook -from airflow.utils.decorators import apply_defaults class EmrCreateJobFlowOperator(BaseOperator): @@ -46,7 +45,6 @@ class EmrCreateJobFlowOperator(BaseOperator): template_fields_renderers = {"job_flow_overrides": "json"} ui_color = '#f9c915' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/emr_modify_cluster.py b/airflow/providers/amazon/aws/operators/emr_modify_cluster.py index a04e845ac551b..9a25b9c69ecde 100644 --- a/airflow/providers/amazon/aws/operators/emr_modify_cluster.py +++ b/airflow/providers/amazon/aws/operators/emr_modify_cluster.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.emr import EmrHook -from airflow.utils.decorators import apply_defaults class EmrModifyClusterOperator(BaseOperator): @@ -41,7 +40,6 @@ class EmrModifyClusterOperator(BaseOperator): template_ext = () ui_color = '#f9c915' - @apply_defaults def __init__( self, *, cluster_id: str, step_concurrency_level: int, aws_conn_id: str = 'aws_default', **kwargs ): diff --git a/airflow/providers/amazon/aws/operators/emr_terminate_job_flow.py b/airflow/providers/amazon/aws/operators/emr_terminate_job_flow.py index 9d75eaf6e82be..43a72041ab238 100644 --- a/airflow/providers/amazon/aws/operators/emr_terminate_job_flow.py +++ b/airflow/providers/amazon/aws/operators/emr_terminate_job_flow.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.emr import EmrHook -from airflow.utils.decorators import apply_defaults class EmrTerminateJobFlowOperator(BaseOperator): @@ -38,7 +37,6 @@ class EmrTerminateJobFlowOperator(BaseOperator): template_ext = () ui_color = '#f9c915' - @apply_defaults def __init__(self, *, job_flow_id: str, aws_conn_id: str = 'aws_default', **kwargs): super().__init__(**kwargs) self.job_flow_id = job_flow_id diff --git a/airflow/providers/amazon/aws/operators/glacier.py b/airflow/providers/amazon/aws/operators/glacier.py index c88d6526213ad..9fff4b556f7ce 100644 --- a/airflow/providers/amazon/aws/operators/glacier.py +++ b/airflow/providers/amazon/aws/operators/glacier.py @@ -17,7 +17,6 @@ # under the License. from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.glacier import GlacierHook -from airflow.utils.decorators import apply_defaults class GlacierCreateJobOperator(BaseOperator): @@ -36,7 +35,6 @@ class GlacierCreateJobOperator(BaseOperator): template_fields = ("vault_name",) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/glue.py b/airflow/providers/amazon/aws/operators/glue.py index 855a6a564a8bb..254457932d4a2 100644 --- a/airflow/providers/amazon/aws/operators/glue.py +++ b/airflow/providers/amazon/aws/operators/glue.py @@ -22,7 +22,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.glue import AwsGlueJobHook from airflow.providers.amazon.aws.hooks.s3 import S3Hook -from airflow.utils.decorators import apply_defaults class AwsGlueJobOperator(BaseOperator): @@ -60,7 +59,6 @@ class AwsGlueJobOperator(BaseOperator): template_fields_renderers = {"script_args": "py"} ui_color = '#ededed' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/glue_crawler.py b/airflow/providers/amazon/aws/operators/glue_crawler.py index 586c89a7d5836..c760215e8c215 100644 --- a/airflow/providers/amazon/aws/operators/glue_crawler.py +++ b/airflow/providers/amazon/aws/operators/glue_crawler.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.glue_crawler import AwsGlueCrawlerHook -from airflow.utils.decorators import apply_defaults class AwsGlueCrawlerOperator(BaseOperator): @@ -42,7 +41,6 @@ class AwsGlueCrawlerOperator(BaseOperator): ui_color = '#ededed' - @apply_defaults def __init__( self, config, diff --git a/airflow/providers/amazon/aws/operators/s3_bucket.py b/airflow/providers/amazon/aws/operators/s3_bucket.py index d7c0cc727339a..0cf2307d6b1ed 100644 --- a/airflow/providers/amazon/aws/operators/s3_bucket.py +++ b/airflow/providers/amazon/aws/operators/s3_bucket.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook -from airflow.utils.decorators import apply_defaults class S3CreateBucketOperator(BaseOperator): @@ -45,7 +44,6 @@ class S3CreateBucketOperator(BaseOperator): template_fields = ("bucket_name",) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/s3_copy_object.py b/airflow/providers/amazon/aws/operators/s3_copy_object.py index 48bc52bc194ff..ccc9102e57504 100644 --- a/airflow/providers/amazon/aws/operators/s3_copy_object.py +++ b/airflow/providers/amazon/aws/operators/s3_copy_object.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook -from airflow.utils.decorators import apply_defaults class S3CopyObjectOperator(BaseOperator): @@ -70,7 +69,6 @@ class S3CopyObjectOperator(BaseOperator): template_fields = ('source_bucket_key', 'dest_bucket_key', 'source_bucket_name', 'dest_bucket_name') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/s3_delete_objects.py b/airflow/providers/amazon/aws/operators/s3_delete_objects.py index 96c9e14488e3e..de8d8ce620a07 100644 --- a/airflow/providers/amazon/aws/operators/s3_delete_objects.py +++ b/airflow/providers/amazon/aws/operators/s3_delete_objects.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook -from airflow.utils.decorators import apply_defaults class S3DeleteObjectsOperator(BaseOperator): @@ -62,7 +61,6 @@ class S3DeleteObjectsOperator(BaseOperator): template_fields = ('keys', 'bucket', 'prefix') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/s3_file_transform.py b/airflow/providers/amazon/aws/operators/s3_file_transform.py index 1084e2bc764f7..3ab861f7ec6ce 100644 --- a/airflow/providers/amazon/aws/operators/s3_file_transform.py +++ b/airflow/providers/amazon/aws/operators/s3_file_transform.py @@ -24,7 +24,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook -from airflow.utils.decorators import apply_defaults class S3FileTransformOperator(BaseOperator): @@ -82,7 +81,6 @@ class S3FileTransformOperator(BaseOperator): template_ext = () ui_color = '#f9c915' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/s3_list.py b/airflow/providers/amazon/aws/operators/s3_list.py index 58d599d8a3eb2..4d610a302e02a 100644 --- a/airflow/providers/amazon/aws/operators/s3_list.py +++ b/airflow/providers/amazon/aws/operators/s3_list.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook -from airflow.utils.decorators import apply_defaults class S3ListOperator(BaseOperator): @@ -69,7 +68,6 @@ class S3ListOperator(BaseOperator): template_fields: Iterable[str] = ('bucket', 'prefix', 'delimiter') ui_color = '#ffd700' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/sagemaker_base.py b/airflow/providers/amazon/aws/operators/sagemaker_base.py index 86b347bed7a90..8c414c1250373 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker_base.py +++ b/airflow/providers/amazon/aws/operators/sagemaker_base.py @@ -26,7 +26,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook -from airflow.utils.decorators import apply_defaults class SageMakerBaseOperator(BaseOperator): @@ -46,7 +45,6 @@ class SageMakerBaseOperator(BaseOperator): integer_fields = [] # type: Iterable[Iterable[str]] - @apply_defaults def __init__(self, *, config: dict, aws_conn_id: str = 'aws_default', **kwargs): super().__init__(**kwargs) diff --git a/airflow/providers/amazon/aws/operators/sagemaker_endpoint.py b/airflow/providers/amazon/aws/operators/sagemaker_endpoint.py index 35b0b11f897b8..352c88d913e1f 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker_endpoint.py +++ b/airflow/providers/amazon/aws/operators/sagemaker_endpoint.py @@ -22,7 +22,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.operators.sagemaker_base import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults class SageMakerEndpointOperator(SageMakerBaseOperator): @@ -71,7 +70,6 @@ class SageMakerEndpointOperator(SageMakerBaseOperator): :type operation: str """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/sagemaker_endpoint_config.py b/airflow/providers/amazon/aws/operators/sagemaker_endpoint_config.py index a2add7b3e0261..448a1a5310027 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker_endpoint_config.py +++ b/airflow/providers/amazon/aws/operators/sagemaker_endpoint_config.py @@ -18,7 +18,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.operators.sagemaker_base import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults class SageMakerEndpointConfigOperator(SageMakerBaseOperator): @@ -37,7 +36,6 @@ class SageMakerEndpointConfigOperator(SageMakerBaseOperator): integer_fields = [['ProductionVariants', 'InitialInstanceCount']] - @apply_defaults def __init__(self, *, config: dict, **kwargs): super().__init__(config=config, **kwargs) diff --git a/airflow/providers/amazon/aws/operators/sagemaker_model.py b/airflow/providers/amazon/aws/operators/sagemaker_model.py index 0e8cbf4773475..c29f8a99f83d8 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker_model.py +++ b/airflow/providers/amazon/aws/operators/sagemaker_model.py @@ -19,7 +19,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.operators.sagemaker_base import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults class SageMakerModelOperator(SageMakerBaseOperator): @@ -36,7 +35,6 @@ class SageMakerModelOperator(SageMakerBaseOperator): :type aws_conn_id: str """ - @apply_defaults def __init__(self, *, config, **kwargs): super().__init__(config=config, **kwargs) diff --git a/airflow/providers/amazon/aws/operators/sagemaker_processing.py b/airflow/providers/amazon/aws/operators/sagemaker_processing.py index 271b46bd54bc5..3231fc7d2e7f7 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker_processing.py +++ b/airflow/providers/amazon/aws/operators/sagemaker_processing.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.operators.sagemaker_base import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults class SageMakerProcessingOperator(SageMakerBaseOperator): @@ -52,7 +51,6 @@ class SageMakerProcessingOperator(SageMakerBaseOperator): :type action_if_job_exists: str """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/sagemaker_training.py b/airflow/providers/amazon/aws/operators/sagemaker_training.py index 7d9eaf236ac6b..c748666c00573 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker_training.py +++ b/airflow/providers/amazon/aws/operators/sagemaker_training.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.operators.sagemaker_base import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults class SageMakerTrainingOperator(SageMakerBaseOperator): @@ -58,7 +57,6 @@ class SageMakerTrainingOperator(SageMakerBaseOperator): ['StoppingCondition', 'MaxRuntimeInSeconds'], ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/sagemaker_transform.py b/airflow/providers/amazon/aws/operators/sagemaker_transform.py index b264d2df137e6..7449e865b6be9 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker_transform.py +++ b/airflow/providers/amazon/aws/operators/sagemaker_transform.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.operators.sagemaker_base import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults class SageMakerTransformOperator(SageMakerBaseOperator): @@ -62,7 +61,6 @@ class SageMakerTransformOperator(SageMakerBaseOperator): :type max_ingestion_time: int """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/sagemaker_tuning.py b/airflow/providers/amazon/aws/operators/sagemaker_tuning.py index 38664a8b1bef6..2bb38578d6130 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker_tuning.py +++ b/airflow/providers/amazon/aws/operators/sagemaker_tuning.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.operators.sagemaker_base import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults class SageMakerTuningOperator(SageMakerBaseOperator): @@ -55,7 +54,6 @@ class SageMakerTuningOperator(SageMakerBaseOperator): ['TrainingJobDefinition', 'StoppingCondition', 'MaxRuntimeInSeconds'], ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/sns.py b/airflow/providers/amazon/aws/operators/sns.py index 1912bba7074f1..0cd0a72db0e75 100644 --- a/airflow/providers/amazon/aws/operators/sns.py +++ b/airflow/providers/amazon/aws/operators/sns.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.sns import AwsSnsHook -from airflow.utils.decorators import apply_defaults class SnsPublishOperator(BaseOperator): @@ -45,7 +44,6 @@ class SnsPublishOperator(BaseOperator): template_ext = () template_fields_renderers = {"message_attributes": "py"} - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/sqs.py b/airflow/providers/amazon/aws/operators/sqs.py index afc50e45c2ad4..c7ac46ce2e1fe 100644 --- a/airflow/providers/amazon/aws/operators/sqs.py +++ b/airflow/providers/amazon/aws/operators/sqs.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.sqs import SQSHook -from airflow.utils.decorators import apply_defaults class SQSPublishOperator(BaseOperator): @@ -43,7 +42,6 @@ class SQSPublishOperator(BaseOperator): template_fields = ('sqs_queue', 'message_content', 'delay_seconds') ui_color = '#6ad3fa' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/step_function_get_execution_output.py b/airflow/providers/amazon/aws/operators/step_function_get_execution_output.py index 769f06ca951ae..770ca5266346a 100644 --- a/airflow/providers/amazon/aws/operators/step_function_get_execution_output.py +++ b/airflow/providers/amazon/aws/operators/step_function_get_execution_output.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.step_function import StepFunctionHook -from airflow.utils.decorators import apply_defaults class StepFunctionGetExecutionOutputOperator(BaseOperator): @@ -42,7 +41,6 @@ class StepFunctionGetExecutionOutputOperator(BaseOperator): template_ext = () ui_color = '#f9c915' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/operators/step_function_start_execution.py b/airflow/providers/amazon/aws/operators/step_function_start_execution.py index b364ba559f74c..8507e3b3c061b 100644 --- a/airflow/providers/amazon/aws/operators/step_function_start_execution.py +++ b/airflow/providers/amazon/aws/operators/step_function_start_execution.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.step_function import StepFunctionHook -from airflow.utils.decorators import apply_defaults class StepFunctionStartExecutionOperator(BaseOperator): @@ -48,7 +47,6 @@ class StepFunctionStartExecutionOperator(BaseOperator): template_ext = () ui_color = '#f9c915' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/athena.py b/airflow/providers/amazon/aws/sensors/athena.py index 8d731e8f23f45..d93010cd975f1 100644 --- a/airflow/providers/amazon/aws/sensors/athena.py +++ b/airflow/providers/amazon/aws/sensors/athena.py @@ -25,7 +25,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.athena import AWSAthenaHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class AthenaSensor(BaseSensorOperator): @@ -59,7 +58,6 @@ class AthenaSensor(BaseSensorOperator): template_ext = () ui_color = '#66c3ff' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/cloud_formation.py b/airflow/providers/amazon/aws/sensors/cloud_formation.py index 7470fae0a79b4..19769997cf28b 100644 --- a/airflow/providers/amazon/aws/sensors/cloud_formation.py +++ b/airflow/providers/amazon/aws/sensors/cloud_formation.py @@ -25,7 +25,6 @@ from airflow.providers.amazon.aws.hooks.cloud_formation import AWSCloudFormationHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class CloudFormationCreateStackSensor(BaseSensorOperator): @@ -44,7 +43,6 @@ class CloudFormationCreateStackSensor(BaseSensorOperator): template_fields = ['stack_name'] ui_color = '#C5CAE9' - @apply_defaults def __init__(self, *, stack_name, aws_conn_id='aws_default', region_name=None, **kwargs): super().__init__(**kwargs) self.stack_name = stack_name @@ -81,7 +79,6 @@ class CloudFormationDeleteStackSensor(BaseSensorOperator): template_fields = ['stack_name'] ui_color = '#C5CAE9' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/ec2_instance_state.py b/airflow/providers/amazon/aws/sensors/ec2_instance_state.py index f3fbd9f9a777c..83c12d413876e 100644 --- a/airflow/providers/amazon/aws/sensors/ec2_instance_state.py +++ b/airflow/providers/amazon/aws/sensors/ec2_instance_state.py @@ -21,7 +21,6 @@ from airflow.providers.amazon.aws.hooks.ec2 import EC2Hook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class EC2InstanceStateSensor(BaseSensorOperator): @@ -42,7 +41,6 @@ class EC2InstanceStateSensor(BaseSensorOperator): ui_fgcolor = "#ffffff" valid_states = ["running", "stopped", "terminated"] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/emr_base.py b/airflow/providers/amazon/aws/sensors/emr_base.py index 83eb31f554189..b2462d4905bab 100644 --- a/airflow/providers/amazon/aws/sensors/emr_base.py +++ b/airflow/providers/amazon/aws/sensors/emr_base.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.emr import EmrHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class EmrBaseSensor(BaseSensorOperator): @@ -41,7 +40,6 @@ class EmrBaseSensor(BaseSensorOperator): ui_color = '#66c3ff' - @apply_defaults def __init__(self, *, aws_conn_id: str = 'aws_default', **kwargs): super().__init__(**kwargs) self.aws_conn_id = aws_conn_id diff --git a/airflow/providers/amazon/aws/sensors/emr_job_flow.py b/airflow/providers/amazon/aws/sensors/emr_job_flow.py index c08e9db92675a..a35d4dba5bd7c 100644 --- a/airflow/providers/amazon/aws/sensors/emr_job_flow.py +++ b/airflow/providers/amazon/aws/sensors/emr_job_flow.py @@ -19,7 +19,6 @@ from typing import Any, Dict, Iterable, Optional from airflow.providers.amazon.aws.sensors.emr_base import EmrBaseSensor -from airflow.utils.decorators import apply_defaults class EmrJobFlowSensor(EmrBaseSensor): @@ -45,7 +44,6 @@ class EmrJobFlowSensor(EmrBaseSensor): template_fields = ['job_flow_id', 'target_states', 'failed_states'] template_ext = () - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/emr_step.py b/airflow/providers/amazon/aws/sensors/emr_step.py index f3c3d593fb73c..b3db376610e20 100644 --- a/airflow/providers/amazon/aws/sensors/emr_step.py +++ b/airflow/providers/amazon/aws/sensors/emr_step.py @@ -19,7 +19,6 @@ from typing import Any, Dict, Iterable, Optional from airflow.providers.amazon.aws.sensors.emr_base import EmrBaseSensor -from airflow.utils.decorators import apply_defaults class EmrStepSensor(EmrBaseSensor): @@ -44,7 +43,6 @@ class EmrStepSensor(EmrBaseSensor): template_fields = ['job_flow_id', 'step_id', 'target_states', 'failed_states'] template_ext = () - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/glacier.py b/airflow/providers/amazon/aws/sensors/glacier.py index 6ba5167674dea..6c45969384d71 100644 --- a/airflow/providers/amazon/aws/sensors/glacier.py +++ b/airflow/providers/amazon/aws/sensors/glacier.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.glacier import GlacierHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class JobStatus(Enum): @@ -65,7 +64,6 @@ class GlacierJobOperationSensor(BaseSensorOperator): template_fields = ["vault_name", "job_id"] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/glue.py b/airflow/providers/amazon/aws/sensors/glue.py index 21a82da9ee9d0..2cc49c4cdf18a 100644 --- a/airflow/providers/amazon/aws/sensors/glue.py +++ b/airflow/providers/amazon/aws/sensors/glue.py @@ -19,7 +19,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.glue import AwsGlueJobHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class AwsGlueJobSensor(BaseSensorOperator): @@ -35,7 +34,6 @@ class AwsGlueJobSensor(BaseSensorOperator): template_fields = ('job_name', 'run_id') - @apply_defaults def __init__(self, *, job_name: str, run_id: str, aws_conn_id: str = 'aws_default', **kwargs): super().__init__(**kwargs) self.job_name = job_name diff --git a/airflow/providers/amazon/aws/sensors/glue_catalog_partition.py b/airflow/providers/amazon/aws/sensors/glue_catalog_partition.py index 240d0781ea3b6..1af865402f6a1 100644 --- a/airflow/providers/amazon/aws/sensors/glue_catalog_partition.py +++ b/airflow/providers/amazon/aws/sensors/glue_catalog_partition.py @@ -19,7 +19,6 @@ from airflow.providers.amazon.aws.hooks.glue_catalog import AwsGlueCatalogHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class AwsGlueCatalogPartitionSensor(BaseSensorOperator): @@ -56,7 +55,6 @@ class AwsGlueCatalogPartitionSensor(BaseSensorOperator): ) ui_color = '#C5CAE9' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/glue_crawler.py b/airflow/providers/amazon/aws/sensors/glue_crawler.py index 4ecd4a0c1ea64..fdea9e07fd3ee 100644 --- a/airflow/providers/amazon/aws/sensors/glue_crawler.py +++ b/airflow/providers/amazon/aws/sensors/glue_crawler.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.glue_crawler import AwsGlueCrawlerHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class AwsGlueCrawlerSensor(BaseSensorOperator): @@ -34,7 +33,6 @@ class AwsGlueCrawlerSensor(BaseSensorOperator): :type aws_conn_id: str """ - @apply_defaults def __init__(self, *, crawler_name: str, aws_conn_id: str = 'aws_default', **kwargs) -> None: super().__init__(**kwargs) self.crawler_name = crawler_name diff --git a/airflow/providers/amazon/aws/sensors/redshift.py b/airflow/providers/amazon/aws/sensors/redshift.py index 508b6c0bd4a44..669e6c0c86b89 100644 --- a/airflow/providers/amazon/aws/sensors/redshift.py +++ b/airflow/providers/amazon/aws/sensors/redshift.py @@ -19,7 +19,6 @@ from airflow.providers.amazon.aws.hooks.redshift import RedshiftHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class AwsRedshiftClusterSensor(BaseSensorOperator): @@ -34,7 +33,6 @@ class AwsRedshiftClusterSensor(BaseSensorOperator): template_fields = ('cluster_identifier', 'target_status') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/s3_key.py b/airflow/providers/amazon/aws/sensors/s3_key.py index bd665a80edc68..e61e8d17727b4 100644 --- a/airflow/providers/amazon/aws/sensors/s3_key.py +++ b/airflow/providers/amazon/aws/sensors/s3_key.py @@ -22,7 +22,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class S3KeySensor(BaseSensorOperator): @@ -58,7 +57,6 @@ class S3KeySensor(BaseSensorOperator): template_fields = ('bucket_key', 'bucket_name') - @apply_defaults def __init__( self, *, @@ -151,7 +149,6 @@ def check_fn(self, data: List) -> bool: :type check_fn: Optional[Callable[..., bool]] """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/s3_keys_unchanged.py b/airflow/providers/amazon/aws/sensors/s3_keys_unchanged.py index 8c3a4cdffc43b..c74eff59d3526 100644 --- a/airflow/providers/amazon/aws/sensors/s3_keys_unchanged.py +++ b/airflow/providers/amazon/aws/sensors/s3_keys_unchanged.py @@ -27,7 +27,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.sensors.base import BaseSensorOperator, poke_mode_only -from airflow.utils.decorators import apply_defaults @poke_mode_only @@ -74,7 +73,6 @@ class S3KeysUnchangedSensor(BaseSensorOperator): template_fields = ('bucket_name', 'prefix') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/s3_prefix.py b/airflow/providers/amazon/aws/sensors/s3_prefix.py index aacc527c9b776..2fa923a1b21b7 100644 --- a/airflow/providers/amazon/aws/sensors/s3_prefix.py +++ b/airflow/providers/amazon/aws/sensors/s3_prefix.py @@ -19,7 +19,6 @@ from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class S3PrefixSensor(BaseSensorOperator): @@ -55,7 +54,6 @@ class S3PrefixSensor(BaseSensorOperator): template_fields = ('prefix', 'bucket_name') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/sagemaker_base.py b/airflow/providers/amazon/aws/sensors/sagemaker_base.py index 657212209093a..fe2af29aba49e 100644 --- a/airflow/providers/amazon/aws/sensors/sagemaker_base.py +++ b/airflow/providers/amazon/aws/sensors/sagemaker_base.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class SageMakerBaseSensor(BaseSensorOperator): @@ -33,7 +32,6 @@ class SageMakerBaseSensor(BaseSensorOperator): ui_color = '#ededed' - @apply_defaults def __init__(self, *, aws_conn_id: str = 'aws_default', **kwargs): super().__init__(**kwargs) self.aws_conn_id = aws_conn_id diff --git a/airflow/providers/amazon/aws/sensors/sagemaker_endpoint.py b/airflow/providers/amazon/aws/sensors/sagemaker_endpoint.py index 1a1b6f73ca97c..bb3885f0520d3 100644 --- a/airflow/providers/amazon/aws/sensors/sagemaker_endpoint.py +++ b/airflow/providers/amazon/aws/sensors/sagemaker_endpoint.py @@ -18,7 +18,6 @@ from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook from airflow.providers.amazon.aws.sensors.sagemaker_base import SageMakerBaseSensor -from airflow.utils.decorators import apply_defaults class SageMakerEndpointSensor(SageMakerBaseSensor): @@ -33,7 +32,6 @@ class SageMakerEndpointSensor(SageMakerBaseSensor): template_fields = ['endpoint_name'] template_ext = () - @apply_defaults def __init__(self, *, endpoint_name, **kwargs): super().__init__(**kwargs) self.endpoint_name = endpoint_name diff --git a/airflow/providers/amazon/aws/sensors/sagemaker_training.py b/airflow/providers/amazon/aws/sensors/sagemaker_training.py index c1491a84f9861..12e1264205ff6 100644 --- a/airflow/providers/amazon/aws/sensors/sagemaker_training.py +++ b/airflow/providers/amazon/aws/sensors/sagemaker_training.py @@ -20,7 +20,6 @@ from airflow.providers.amazon.aws.hooks.sagemaker import LogState, SageMakerHook from airflow.providers.amazon.aws.sensors.sagemaker_base import SageMakerBaseSensor -from airflow.utils.decorators import apply_defaults class SageMakerTrainingSensor(SageMakerBaseSensor): @@ -37,7 +36,6 @@ class SageMakerTrainingSensor(SageMakerBaseSensor): template_fields = ['job_name'] template_ext = () - @apply_defaults def __init__(self, *, job_name, print_log=True, **kwargs): super().__init__(**kwargs) self.job_name = job_name diff --git a/airflow/providers/amazon/aws/sensors/sagemaker_transform.py b/airflow/providers/amazon/aws/sensors/sagemaker_transform.py index a751e5638877e..6e03066158b8a 100644 --- a/airflow/providers/amazon/aws/sensors/sagemaker_transform.py +++ b/airflow/providers/amazon/aws/sensors/sagemaker_transform.py @@ -18,7 +18,6 @@ from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook from airflow.providers.amazon.aws.sensors.sagemaker_base import SageMakerBaseSensor -from airflow.utils.decorators import apply_defaults class SageMakerTransformSensor(SageMakerBaseSensor): @@ -34,7 +33,6 @@ class SageMakerTransformSensor(SageMakerBaseSensor): template_fields = ['job_name'] template_ext = () - @apply_defaults def __init__(self, *, job_name: str, **kwargs): super().__init__(**kwargs) self.job_name = job_name diff --git a/airflow/providers/amazon/aws/sensors/sagemaker_tuning.py b/airflow/providers/amazon/aws/sensors/sagemaker_tuning.py index 96080e08d4285..9f05fa87b7b4b 100644 --- a/airflow/providers/amazon/aws/sensors/sagemaker_tuning.py +++ b/airflow/providers/amazon/aws/sensors/sagemaker_tuning.py @@ -18,7 +18,6 @@ from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook from airflow.providers.amazon.aws.sensors.sagemaker_base import SageMakerBaseSensor -from airflow.utils.decorators import apply_defaults class SageMakerTuningSensor(SageMakerBaseSensor): @@ -34,7 +33,6 @@ class SageMakerTuningSensor(SageMakerBaseSensor): template_fields = ['job_name'] template_ext = () - @apply_defaults def __init__(self, *, job_name: str, **kwargs): super().__init__(**kwargs) self.job_name = job_name diff --git a/airflow/providers/amazon/aws/sensors/sqs.py b/airflow/providers/amazon/aws/sensors/sqs.py index c07b2052dbf12..dc6217b5bd0a3 100644 --- a/airflow/providers/amazon/aws/sensors/sqs.py +++ b/airflow/providers/amazon/aws/sensors/sqs.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.sqs import SQSHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class SQSSensor(BaseSensorOperator): @@ -42,7 +41,6 @@ class SQSSensor(BaseSensorOperator): template_fields = ('sqs_queue', 'max_messages') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/sensors/step_function_execution.py b/airflow/providers/amazon/aws/sensors/step_function_execution.py index 387a2d8145436..39cd74f69397d 100644 --- a/airflow/providers/amazon/aws/sensors/step_function_execution.py +++ b/airflow/providers/amazon/aws/sensors/step_function_execution.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.step_function import StepFunctionHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class StepFunctionExecutionSensor(BaseSensorOperator): @@ -51,7 +50,6 @@ class StepFunctionExecutionSensor(BaseSensorOperator): template_ext = () ui_color = '#66c3ff' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py b/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py index 9c63278231ad4..7e0e41062cafc 100644 --- a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py @@ -31,7 +31,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.dynamodb import AwsDynamoDBHook from airflow.providers.amazon.aws.hooks.s3 import S3Hook -from airflow.utils.decorators import apply_defaults def _convert_item_to_json_bytes(item: Dict[str, Any]) -> bytes: @@ -97,7 +96,6 @@ class DynamoDBToS3Operator(BaseOperator): :type process_func: Callable[[Dict[str, Any]], bytes] """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/transfers/exasol_to_s3.py b/airflow/providers/amazon/aws/transfers/exasol_to_s3.py index d2c62ac1181f9..678429262a732 100644 --- a/airflow/providers/amazon/aws/transfers/exasol_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/exasol_to_s3.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.exasol.hooks.exasol import ExasolHook -from airflow.utils.decorators import apply_defaults class ExasolToS3Operator(BaseOperator): @@ -61,7 +60,6 @@ class ExasolToS3Operator(BaseOperator): template_ext = ('.sql',) ui_color = '#ededed' - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/amazon/aws/transfers/ftp_to_s3.py b/airflow/providers/amazon/aws/transfers/ftp_to_s3.py index 14f43da99e215..d2a28c86d7b09 100644 --- a/airflow/providers/amazon/aws/transfers/ftp_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/ftp_to_s3.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.ftp.hooks.ftp import FTPHook -from airflow.utils.decorators import apply_defaults class FTPToS3Operator(BaseOperator): @@ -60,7 +59,6 @@ class FTPToS3Operator(BaseOperator): 'ftp_path', ) - @apply_defaults def __init__( self, s3_bucket, diff --git a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py b/airflow/providers/amazon/aws/transfers/gcs_to_s3.py index 58b2e7c884c6a..0684305ba277e 100644 --- a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/gcs_to_s3.py @@ -22,7 +22,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults class GCSToS3Operator(BaseOperator): @@ -94,7 +93,6 @@ class GCSToS3Operator(BaseOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, # pylint: disable=too-many-arguments diff --git a/airflow/providers/amazon/aws/transfers/glacier_to_gcs.py b/airflow/providers/amazon/aws/transfers/glacier_to_gcs.py index f9366662838cb..ca1bf29b16941 100644 --- a/airflow/providers/amazon/aws/transfers/glacier_to_gcs.py +++ b/airflow/providers/amazon/aws/transfers/glacier_to_gcs.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.glacier import GlacierHook from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults class GlacierToGCSOperator(BaseOperator): @@ -68,7 +67,6 @@ class GlacierToGCSOperator(BaseOperator): template_fields = ("vault_name", "bucket_name", "object_name") - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/transfers/google_api_to_s3.py b/airflow/providers/amazon/aws/transfers/google_api_to_s3.py index 1951a392419fd..1c194285c5612 100644 --- a/airflow/providers/amazon/aws/transfers/google_api_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/google_api_to_s3.py @@ -25,7 +25,6 @@ from airflow.models.xcom import MAX_XCOM_SIZE, XCOM_RETURN_KEY from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.google.common.hooks.discovery_api import GoogleDiscoveryApiHook -from airflow.utils.decorators import apply_defaults class GoogleApiToS3Operator(BaseOperator): @@ -101,7 +100,6 @@ class GoogleApiToS3Operator(BaseOperator): template_ext = () ui_color = '#cc181e' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py b/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py index 2939be57a2096..5f027b58f6586 100644 --- a/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py +++ b/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py @@ -24,7 +24,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.dynamodb import AwsDynamoDBHook from airflow.providers.apache.hive.hooks.hive import HiveServer2Hook -from airflow.utils.decorators import apply_defaults class HiveToDynamoDBOperator(BaseOperator): @@ -49,7 +48,8 @@ class HiveToDynamoDBOperator(BaseOperator): :type region_name: str :param schema: hive database schema :type schema: str - :param hiveserver2_conn_id: source hive connection + :param hiveserver2_conn_id: Reference to the + :ref: `Hive Server2 thrift service connection id `. :type hiveserver2_conn_id: str :param aws_conn_id: aws connection :type aws_conn_id: str @@ -59,7 +59,6 @@ class HiveToDynamoDBOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#a0e08c' - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py b/airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py index 015e7342c79ef..cb65bf112e80f 100644 --- a/airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.imap.hooks.imap import ImapHook -from airflow.utils.decorators import apply_defaults class ImapAttachmentToS3Operator(BaseOperator): @@ -51,7 +50,6 @@ class ImapAttachmentToS3Operator(BaseOperator): template_fields = ('imap_attachment_name', 's3_key', 'imap_mail_filter') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/transfers/mongo_to_s3.py b/airflow/providers/amazon/aws/transfers/mongo_to_s3.py index 0dc24dcae39b0..b88036d6fb160 100644 --- a/airflow/providers/amazon/aws/transfers/mongo_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/mongo_to_s3.py @@ -24,7 +24,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.mongo.hooks.mongo import MongoHook -from airflow.utils.decorators import apply_defaults _DEPRECATION_MSG = ( "The s3_conn_id parameter has been deprecated. You should pass instead the aws_conn_id parameter." @@ -41,7 +40,7 @@ class MongoToS3Operator(BaseOperator): :param mongo_collection: reference to a specific collection in your mongo db :type mongo_collection: str :param mongo_query: query to execute. A list including a dict of the query - :type mongo_query: list + :type mongo_query: Union[list, dict] :param s3_bucket: reference to a specific S3 bucket to store the data :type s3_bucket: str :param s3_key: in which S3 key the file will be stored @@ -50,8 +49,8 @@ class MongoToS3Operator(BaseOperator): :type mongo_db: str :param replace: whether or not to replace the file in S3 if it previously existed :type replace: bool - :param allow_disk_use: in the case you are retrieving a lot of data, you may have - to use the disk to save it instead of saving all in the RAM + :param allow_disk_use: enables writing to temporary files in the case you are handling large dataset. + This only takes effect when `mongo_query` is a list - running an aggregate pipeline :type allow_disk_use: bool :param compression: type of compression to use for output file in S3. Currently only gzip is supported. :type compression: str @@ -62,7 +61,6 @@ class MongoToS3Operator(BaseOperator): template_fields_renderers = {"mongo_query": "py"} # pylint: disable=too-many-instance-attributes - @apply_defaults def __init__( self, *, @@ -117,7 +115,6 @@ def execute(self, context) -> bool: mongo_collection=self.mongo_collection, query=cast(dict, self.mongo_query), mongo_db=self.mongo_db, - allowDiskUse=self.allow_disk_use, ) # Performs transform then stringifies the docs results into json format diff --git a/airflow/providers/amazon/aws/transfers/mysql_to_s3.py b/airflow/providers/amazon/aws/transfers/mysql_to_s3.py index 0cec8eaa87324..223d6bc52270b 100644 --- a/airflow/providers/amazon/aws/transfers/mysql_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/mysql_to_s3.py @@ -27,7 +27,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.mysql.hooks.mysql import MySqlHook -from airflow.utils.decorators import apply_defaults class MySQLToS3Operator(BaseOperator): @@ -71,7 +70,6 @@ class MySQLToS3Operator(BaseOperator): template_ext = ('.sql',) template_fields_renderers = {"query": "sql"} - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/transfers/redshift_to_s3.py b/airflow/providers/amazon/aws/transfers/redshift_to_s3.py index 624960179cf6a..838d22df4db8b 100644 --- a/airflow/providers/amazon/aws/transfers/redshift_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/redshift_to_s3.py @@ -22,7 +22,6 @@ from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.amazon.aws.utils.redshift import build_credentials_block from airflow.providers.postgres.hooks.postgres import PostgresHook -from airflow.utils.decorators import apply_defaults class RedshiftToS3Operator(BaseOperator): @@ -76,7 +75,6 @@ class RedshiftToS3Operator(BaseOperator): template_ext = () ui_color = '#ededed' - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index 86f75d7dd6d95..3dfae24afcde3 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.ftp.hooks.ftp import FTPHook -from airflow.utils.decorators import apply_defaults class S3ToFTPOperator(BaseOperator): @@ -44,7 +43,6 @@ class S3ToFTPOperator(BaseOperator): template_fields = ('s3_bucket', 's3_key') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/transfers/s3_to_redshift.py b/airflow/providers/amazon/aws/transfers/s3_to_redshift.py index 2c8de2659b3f7..14be6126f6c20 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_redshift.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_redshift.py @@ -21,7 +21,6 @@ from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.amazon.aws.utils.redshift import build_credentials_block from airflow.providers.postgres.hooks.postgres import PostgresHook -from airflow.utils.decorators import apply_defaults class S3ToRedshiftOperator(BaseOperator): @@ -68,7 +67,6 @@ class S3ToRedshiftOperator(BaseOperator): template_ext = () ui_color = '#99e699' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/transfers/s3_to_sftp.py b/airflow/providers/amazon/aws/transfers/s3_to_sftp.py index c94ff5f6c5915..486af07da6037 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_sftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_sftp.py @@ -22,7 +22,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.ssh.hooks.ssh import SSHHook -from airflow.utils.decorators import apply_defaults class S3ToSFTPOperator(BaseOperator): @@ -48,7 +47,6 @@ class S3ToSFTPOperator(BaseOperator): template_fields = ('s3_key', 'sftp_path') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/amazon/aws/transfers/sftp_to_s3.py b/airflow/providers/amazon/aws/transfers/sftp_to_s3.py index 482537f0b2880..a3e4d5be070fd 100644 --- a/airflow/providers/amazon/aws/transfers/sftp_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/sftp_to_s3.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.ssh.hooks.ssh import SSHHook -from airflow.utils.decorators import apply_defaults class SFTPToS3Operator(BaseOperator): @@ -48,7 +47,6 @@ class SFTPToS3Operator(BaseOperator): template_fields = ('s3_key', 'sftp_path') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/beam/example_dags/example_beam.py b/airflow/providers/apache/beam/example_dags/example_beam.py index a95ec49535b61..97d02c5ca796e 100644 --- a/airflow/providers/apache/beam/example_dags/example_beam.py +++ b/airflow/providers/apache/beam/example_dags/example_beam.py @@ -34,29 +34,29 @@ from airflow.utils.dates import days_ago GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project') -GCS_INPUT = os.environ.get('APACHE_BEAM_PYTHON', 'gs://apache-beam-samples/shakespeare/kinglear.txt') -GCS_TMP = os.environ.get('APACHE_BEAM_GCS_TMP', 'gs://test-dataflow-example/temp/') -GCS_STAGING = os.environ.get('APACHE_BEAM_GCS_STAGING', 'gs://test-dataflow-example/staging/') -GCS_OUTPUT = os.environ.get('APACHE_BEAM_GCS_OUTPUT', 'gs://test-dataflow-example/output') -GCS_PYTHON = os.environ.get('APACHE_BEAM_PYTHON', 'gs://test-dataflow-example/wordcount_debugging.py') +GCS_INPUT = os.environ.get('APACHE_BEAM_PYTHON', 'gs://INVALID BUCKET NAME/shakespeare/kinglear.txt') +GCS_TMP = os.environ.get('APACHE_BEAM_GCS_TMP', 'gs://INVALID BUCKET NAME/temp/') +GCS_STAGING = os.environ.get('APACHE_BEAM_GCS_STAGING', 'gs://INVALID BUCKET NAME/staging/') +GCS_OUTPUT = os.environ.get('APACHE_BEAM_GCS_OUTPUT', 'gs://INVALID BUCKET NAME/output') +GCS_PYTHON = os.environ.get('APACHE_BEAM_PYTHON', 'gs://INVALID BUCKET NAME/wordcount_debugging.py') GCS_PYTHON_DATAFLOW_ASYNC = os.environ.get( - 'APACHE_BEAM_PYTHON_DATAFLOW_ASYNC', 'gs://test-dataflow-example/wordcount_debugging.py' + 'APACHE_BEAM_PYTHON_DATAFLOW_ASYNC', 'gs://INVALID BUCKET NAME/wordcount_debugging.py' ) GCS_JAR_DIRECT_RUNNER = os.environ.get( 'APACHE_BEAM_DIRECT_RUNNER_JAR', - 'gs://test-dataflow-example/tests/dataflow-templates-bundled-java=11-beam-v2.25.0-DirectRunner.jar', + 'gs://INVALID BUCKET NAME/tests/dataflow-templates-bundled-java=11-beam-v2.25.0-DirectRunner.jar', ) GCS_JAR_DATAFLOW_RUNNER = os.environ.get( - 'APACHE_BEAM_DATAFLOW_RUNNER_JAR', 'gs://test-dataflow-example/word-count-beam-bundled-0.1.jar' + 'APACHE_BEAM_DATAFLOW_RUNNER_JAR', 'gs://INVALID BUCKET NAME/word-count-beam-bundled-0.1.jar' ) GCS_JAR_SPARK_RUNNER = os.environ.get( 'APACHE_BEAM_SPARK_RUNNER_JAR', - 'gs://test-dataflow-example/tests/dataflow-templates-bundled-java=11-beam-v2.25.0-SparkRunner.jar', + 'gs://INVALID BUCKET NAME/tests/dataflow-templates-bundled-java=11-beam-v2.25.0-SparkRunner.jar', ) GCS_JAR_FLINK_RUNNER = os.environ.get( 'APACHE_BEAM_FLINK_RUNNER_JAR', - 'gs://test-dataflow-example/tests/dataflow-templates-bundled-java=11-beam-v2.25.0-FlinkRunner.jar', + 'gs://INVALID BUCKET NAME/tests/dataflow-templates-bundled-java=11-beam-v2.25.0-FlinkRunner.jar', ) GCS_JAR_DIRECT_RUNNER_PARTS = urlparse(GCS_JAR_DIRECT_RUNNER) diff --git a/airflow/providers/apache/beam/operators/beam.py b/airflow/providers/apache/beam/operators/beam.py index 7dd07e923eb89..7ff87bfa02b1a 100644 --- a/airflow/providers/apache/beam/operators/beam.py +++ b/airflow/providers/apache/beam/operators/beam.py @@ -29,7 +29,6 @@ ) from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.cloud.operators.dataflow import CheckJobRunning, DataflowConfiguration -from airflow.utils.decorators import apply_defaults from airflow.utils.helpers import convert_camel_to_snake from airflow.version import version @@ -168,7 +167,6 @@ class BeamRunPythonPipelineOperator(BaseOperator, BeamDataflowMixin): template_fields = ["py_file", "runner", "pipeline_options", "default_pipeline_options", "dataflow_config"] template_fields_renderers = {'dataflow_config': 'json', 'pipeline_options': 'json'} - @apply_defaults def __init__( self, *, @@ -339,7 +337,6 @@ class BeamRunJavaPipelineOperator(BaseOperator, BeamDataflowMixin): template_fields_renderers = {'dataflow_config': 'json', 'pipeline_options': 'json'} ui_color = "#0273d4" - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/cassandra/sensors/record.py b/airflow/providers/apache/cassandra/sensors/record.py index 828b9381f84ce..46e3ffcd00c3b 100644 --- a/airflow/providers/apache/cassandra/sensors/record.py +++ b/airflow/providers/apache/cassandra/sensors/record.py @@ -24,7 +24,6 @@ from airflow.providers.apache.cassandra.hooks.cassandra import CassandraHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class CassandraRecordSensor(BaseSensorOperator): @@ -56,7 +55,6 @@ class CassandraRecordSensor(BaseSensorOperator): template_fields = ('table', 'keys') - @apply_defaults def __init__(self, *, table: str, keys: Dict[str, str], cassandra_conn_id: str, **kwargs: Any) -> None: super().__init__(**kwargs) self.cassandra_conn_id = cassandra_conn_id diff --git a/airflow/providers/apache/cassandra/sensors/table.py b/airflow/providers/apache/cassandra/sensors/table.py index 99f539d8b561f..3521025f12433 100644 --- a/airflow/providers/apache/cassandra/sensors/table.py +++ b/airflow/providers/apache/cassandra/sensors/table.py @@ -25,7 +25,6 @@ from airflow.providers.apache.cassandra.hooks.cassandra import CassandraHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class CassandraTableSensor(BaseSensorOperator): @@ -54,7 +53,6 @@ class CassandraTableSensor(BaseSensorOperator): template_fields = ('table',) - @apply_defaults def __init__(self, *, table: str, cassandra_conn_id: str, **kwargs: Any) -> None: super().__init__(**kwargs) self.cassandra_conn_id = cassandra_conn_id diff --git a/airflow/providers/apache/druid/operators/druid.py b/airflow/providers/apache/druid/operators/druid.py index c2a9290c61e1c..3243806e7ca3a 100644 --- a/airflow/providers/apache/druid/operators/druid.py +++ b/airflow/providers/apache/druid/operators/druid.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.druid.hooks.druid import DruidHook -from airflow.utils.decorators import apply_defaults class DruidOperator(BaseOperator): @@ -37,7 +36,6 @@ class DruidOperator(BaseOperator): template_fields = ('json_index_file',) template_ext = ('.json',) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/druid/operators/druid_check.py b/airflow/providers/apache/druid/operators/druid_check.py index ce16f9b016642..33a4151350e55 100644 --- a/airflow/providers/apache/druid/operators/druid_check.py +++ b/airflow/providers/apache/druid/operators/druid_check.py @@ -31,6 +31,6 @@ def __init__(self, druid_broker_conn_id: str = 'druid_broker_default', **kwargs) """This class is deprecated. Please use `airflow.operators.sql.SQLCheckOperator`.""", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) super().__init__(conn_id=druid_broker_conn_id, **kwargs) diff --git a/airflow/providers/apache/druid/transfers/hive_to_druid.py b/airflow/providers/apache/druid/transfers/hive_to_druid.py index ce775361019c4..f60a91af9602d 100644 --- a/airflow/providers/apache/druid/transfers/hive_to_druid.py +++ b/airflow/providers/apache/druid/transfers/hive_to_druid.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.druid.hooks.druid import DruidHook from airflow.providers.apache.hive.hooks.hive import HiveCliHook, HiveMetastoreHook -from airflow.utils.decorators import apply_defaults LOAD_CHECK_INTERVAL = 5 DEFAULT_TARGET_PARTITION_SIZE = 5000000 @@ -80,7 +79,6 @@ class HiveToDruidOperator(BaseOperator): template_fields = ('sql', 'intervals') template_ext = ('.sql',) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/apache/hdfs/sensors/hdfs.py b/airflow/providers/apache/hdfs/sensors/hdfs.py index c3f87ab1d2f84..fb296eb2fd614 100644 --- a/airflow/providers/apache/hdfs/sensors/hdfs.py +++ b/airflow/providers/apache/hdfs/sensors/hdfs.py @@ -23,7 +23,6 @@ from airflow import settings from airflow.providers.apache.hdfs.hooks.hdfs import HDFSHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults log = logging.getLogger(__name__) @@ -51,7 +50,6 @@ class HdfsSensor(BaseSensorOperator): template_fields = ('filepath',) ui_color = settings.WEB_COLORS['LIGHTBLUE'] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/hdfs/sensors/web_hdfs.py b/airflow/providers/apache/hdfs/sensors/web_hdfs.py index f057d22221b91..5a7eb5eb1ed5c 100644 --- a/airflow/providers/apache/hdfs/sensors/web_hdfs.py +++ b/airflow/providers/apache/hdfs/sensors/web_hdfs.py @@ -18,7 +18,6 @@ from typing import Any, Dict from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class WebHdfsSensor(BaseSensorOperator): @@ -26,7 +25,6 @@ class WebHdfsSensor(BaseSensorOperator): template_fields = ('filepath',) - @apply_defaults def __init__(self, *, filepath: str, webhdfs_conn_id: str = 'webhdfs_default', **kwargs: Any) -> None: super().__init__(**kwargs) self.filepath = filepath diff --git a/airflow/providers/apache/hive/hooks/hive.py b/airflow/providers/apache/hive/hooks/hive.py index c36c626f1d620..3d8d713a72f8e 100644 --- a/airflow/providers/apache/hive/hooks/hive.py +++ b/airflow/providers/apache/hive/hooks/hive.py @@ -68,6 +68,9 @@ class HiveCliHook(BaseHook): The extra connection parameter ``auth`` gets passed as in the ``jdbc`` connection string as is. + :param hive_cli_conn_id: Reference to the + :ref:`Hive CLI connection id `. + :type hive_cli_conn_id: str :param mapred_queue: queue used by the Hadoop Scheduler (Capacity or Fair) :type mapred_queue: str :param mapred_queue_priority: priority within the job queue. @@ -466,7 +469,13 @@ def kill(self) -> None: class HiveMetastoreHook(BaseHook): - """Wrapper to interact with the Hive Metastore""" + """ + Wrapper to interact with the Hive Metastore + + :param metastore_conn_id: reference to the + :ref: `metastore thrift service connection id `. + :type metastore_conn_id: str + """ # java short max val MAX_PART_COUNT = 32767 @@ -729,7 +738,7 @@ def max_partition( :type filter_map: map >>> hh = HiveMetastoreHook() - >>> filter_map = {'ds': '2015-01-01', 'ds': '2014-01-01'} + >>> filter_map = {'ds': '2015-01-01'} >>> t = 'static_babynames_partitioned' >>> hh.max_partition(schema='airflow',\ ... table_name=t, field='ds', filter_map=filter_map) @@ -811,6 +820,12 @@ class HiveServer2Hook(DbApiHook): * the default for run_set_variable_statements is true, if you are using impala you may need to set it to false in the ``extra`` of your connection in the UI + + :param hiveserver2_conn_id: Reference to the + :ref: `Hive Server2 thrift service connection id `. + :type hiveserver2_conn_id: str + :param schema: Hive database name. + :type schema: Optional[str] """ conn_name_attr = 'hiveserver2_conn_id' diff --git a/airflow/providers/apache/hive/operators/hive.py b/airflow/providers/apache/hive/operators/hive.py index 598760fdd6783..a0d8a663daca5 100644 --- a/airflow/providers/apache/hive/operators/hive.py +++ b/airflow/providers/apache/hive/operators/hive.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.hive.hooks.hive import HiveCliHook from airflow.utils import operator_helpers -from airflow.utils.decorators import apply_defaults from airflow.utils.operator_helpers import context_to_airflow_vars @@ -35,7 +34,8 @@ class HiveOperator(BaseOperator): a relative path from the dag file of a (template) hive script. (templated) :type hql: str - :param hive_cli_conn_id: reference to the Hive database. (templated) + :param hive_cli_conn_id: Reference to the + :ref:`Hive CLI connection id `. (templated) :type hive_cli_conn_id: str :param hiveconfs: if defined, these key value pairs will be passed to hive as ``-hiveconf "key"="value"`` @@ -78,7 +78,6 @@ class HiveOperator(BaseOperator): ui_color = '#f0e4ec' # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/hive/operators/hive_stats.py b/airflow/providers/apache/hive/operators/hive_stats.py index 88faa4056ae28..1c08ef131c34d 100644 --- a/airflow/providers/apache/hive/operators/hive_stats.py +++ b/airflow/providers/apache/hive/operators/hive_stats.py @@ -25,7 +25,6 @@ from airflow.providers.apache.hive.hooks.hive import HiveMetastoreHook from airflow.providers.mysql.hooks.mysql import MySqlHook from airflow.providers.presto.hooks.presto import PrestoHook -from airflow.utils.decorators import apply_defaults class HiveStatsCollectionOperator(BaseOperator): @@ -41,6 +40,9 @@ class HiveStatsCollectionOperator(BaseOperator): value BIGINT ); + :param metastore_conn_id: Reference to the + :ref:`Hive Metastore connection id `. + :type metastore_conn_id: str :param table: the source table, in the format ``database.table_name``. (templated) :type table: str :param partition: the source partition. (templated) @@ -62,7 +64,6 @@ class HiveStatsCollectionOperator(BaseOperator): template_fields = ('table', 'partition', 'ds', 'dttm') ui_color = '#aff7a6' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/hive/sensors/hive_partition.py b/airflow/providers/apache/hive/sensors/hive_partition.py index 37b2f86c6b820..3705f812f5712 100644 --- a/airflow/providers/apache/hive/sensors/hive_partition.py +++ b/airflow/providers/apache/hive/sensors/hive_partition.py @@ -19,7 +19,6 @@ from airflow.providers.apache.hive.hooks.hive import HiveMetastoreHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class HivePartitionSensor(BaseSensorOperator): @@ -38,8 +37,8 @@ class HivePartitionSensor(BaseSensorOperator): and apparently supports SQL like notation as in ``ds='2015-01-01' AND type='value'`` and comparison operators as in ``"ds>=2015-01-01"`` :type partition: str - :param metastore_conn_id: reference to the metastore thrift service - connection id + :param metastore_conn_id: reference to the + :ref: `metastore thrift service connection id ` :type metastore_conn_id: str """ @@ -50,7 +49,6 @@ class HivePartitionSensor(BaseSensorOperator): ) ui_color = '#C5CAE9' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/hive/sensors/metastore_partition.py b/airflow/providers/apache/hive/sensors/metastore_partition.py index 2333ef57c0d6e..382de23003008 100644 --- a/airflow/providers/apache/hive/sensors/metastore_partition.py +++ b/airflow/providers/apache/hive/sensors/metastore_partition.py @@ -18,7 +18,6 @@ from typing import Any, Dict from airflow.sensors.sql import SqlSensor -from airflow.utils.decorators import apply_defaults class MetastorePartitionSensor(SqlSensor): @@ -46,7 +45,6 @@ class MetastorePartitionSensor(SqlSensor): ui_color = '#8da7be' poke_context_fields = ('partition_name', 'table', 'schema', 'mysql_conn_id') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/hive/sensors/named_hive_partition.py b/airflow/providers/apache/hive/sensors/named_hive_partition.py index 0a7115871d60e..b497b9a4392bc 100644 --- a/airflow/providers/apache/hive/sensors/named_hive_partition.py +++ b/airflow/providers/apache/hive/sensors/named_hive_partition.py @@ -18,7 +18,6 @@ from typing import Any, Dict, List, Tuple from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class NamedHivePartitionSensor(BaseSensorOperator): @@ -33,8 +32,8 @@ class NamedHivePartitionSensor(BaseSensorOperator): you cannot use logical or comparison operators as in HivePartitionSensor. :type partition_names: list[str] - :param metastore_conn_id: reference to the metastore thrift service - connection id + :param metastore_conn_id: Reference to the + :ref:`metastore thrift service connection id `. :type metastore_conn_id: str """ @@ -42,7 +41,6 @@ class NamedHivePartitionSensor(BaseSensorOperator): ui_color = '#8d99ae' poke_context_fields = ('partition_names', 'metastore_conn_id') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/hive/transfers/hive_to_mysql.py b/airflow/providers/apache/hive/transfers/hive_to_mysql.py index dfd80f8a5ae8a..8464df920273b 100644 --- a/airflow/providers/apache/hive/transfers/hive_to_mysql.py +++ b/airflow/providers/apache/hive/transfers/hive_to_mysql.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.hive.hooks.hive import HiveServer2Hook from airflow.providers.mysql.hooks.mysql import MySqlHook -from airflow.utils.decorators import apply_defaults from airflow.utils.operator_helpers import context_to_airflow_vars @@ -40,8 +39,9 @@ class HiveToMySqlOperator(BaseOperator): :type mysql_table: str :param mysql_conn_id: source mysql connection :type mysql_conn_id: str - :param hiveserver2_conn_id: destination hive connection - :type hiveserver2_conn_id: str + :param metastore_conn_id: Reference to the + :ref:`metastore thrift service connection id `. + :type metastore_conn_id: str :param mysql_preoperator: sql statement to run against mysql prior to import, typically use to truncate of delete in place of the data coming in, allowing the task to be idempotent (running @@ -64,7 +64,6 @@ class HiveToMySqlOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#a0e08c' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/hive/transfers/hive_to_samba.py b/airflow/providers/apache/hive/transfers/hive_to_samba.py index 089ada938a652..a88a8e2b4faa5 100644 --- a/airflow/providers/apache/hive/transfers/hive_to_samba.py +++ b/airflow/providers/apache/hive/transfers/hive_to_samba.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.hive.hooks.hive import HiveServer2Hook from airflow.providers.samba.hooks.samba import SambaHook -from airflow.utils.decorators import apply_defaults from airflow.utils.operator_helpers import context_to_airflow_vars @@ -38,7 +37,8 @@ class HiveToSambaOperator(BaseOperator): :type destination_filepath: str :param samba_conn_id: reference to the samba destination :type samba_conn_id: str - :param hiveserver2_conn_id: reference to the hiveserver2 service + :param hiveserver2_conn_id: Reference to the + :ref: `Hive Server2 thrift service connection id `. :type hiveserver2_conn_id: str """ @@ -48,7 +48,6 @@ class HiveToSambaOperator(BaseOperator): '.sql', ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/hive/transfers/mssql_to_hive.py b/airflow/providers/apache/hive/transfers/mssql_to_hive.py index 3a277ba0c1832..1c404edecd0b7 100644 --- a/airflow/providers/apache/hive/transfers/mssql_to_hive.py +++ b/airflow/providers/apache/hive/transfers/mssql_to_hive.py @@ -28,7 +28,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.hive.hooks.hive import HiveCliHook from airflow.providers.microsoft.mssql.hooks.mssql import MsSqlHook -from airflow.utils.decorators import apply_defaults class MsSqlToHiveOperator(BaseOperator): @@ -63,8 +62,9 @@ class MsSqlToHiveOperator(BaseOperator): :type delimiter: str :param mssql_conn_id: source Microsoft SQL Server connection :type mssql_conn_id: str - :param hive_conn_id: destination hive connection - :type hive_conn_id: str + :param hive_cli_conn_id: Reference to the + :ref:`Hive CLI connection id `. + :type hive_cli_conn_id: str :param tblproperties: TBLPROPERTIES of the hive table being created :type tblproperties: dict """ @@ -73,7 +73,6 @@ class MsSqlToHiveOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#a0e08c' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/hive/transfers/mysql_to_hive.py b/airflow/providers/apache/hive/transfers/mysql_to_hive.py index a1a6dfa90f805..9c828aa32f94b 100644 --- a/airflow/providers/apache/hive/transfers/mysql_to_hive.py +++ b/airflow/providers/apache/hive/transfers/mysql_to_hive.py @@ -28,7 +28,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.hive.hooks.hive import HiveCliHook from airflow.providers.mysql.hooks.mysql import MySqlHook -from airflow.utils.decorators import apply_defaults class MySqlToHiveOperator(BaseOperator): @@ -71,8 +70,9 @@ class MySqlToHiveOperator(BaseOperator): :type escapechar: str :param mysql_conn_id: source mysql connection :type mysql_conn_id: str - :param hive_conn_id: destination hive connection - :type hive_conn_id: str + :param hive_cli_conn_id: Reference to the + :ref:`Hive CLI connection id `. + :type hive_cli_conn_id: str :param tblproperties: TBLPROPERTIES of the hive table being created :type tblproperties: dict """ @@ -81,7 +81,6 @@ class MySqlToHiveOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#a0e08c' - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/apache/hive/transfers/s3_to_hive.py b/airflow/providers/apache/hive/transfers/s3_to_hive.py index 962b92fb2b08a..4e833d73f5c5b 100644 --- a/airflow/providers/apache/hive/transfers/s3_to_hive.py +++ b/airflow/providers/apache/hive/transfers/s3_to_hive.py @@ -30,7 +30,6 @@ from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.apache.hive.hooks.hive import HiveCliHook from airflow.utils.compression import uncompress_file -from airflow.utils.decorators import apply_defaults class S3ToHiveOperator(BaseOperator): # pylint: disable=too-many-instance-attributes @@ -88,7 +87,8 @@ class S3ToHiveOperator(BaseOperator): # pylint: disable=too-many-instance-attri You can specify this argument if you want to use a different CA cert bundle than the one used by botocore. :type verify: bool or str - :param hive_cli_conn_id: destination hive connection + :param hive_cli_conn_id: Reference to the + :ref:`Hive CLI connection id `. :type hive_cli_conn_id: str :param input_compressed: Boolean to determine if file decompression is required to process headers @@ -103,7 +103,6 @@ class S3ToHiveOperator(BaseOperator): # pylint: disable=too-many-instance-attri template_ext = () ui_color = '#a0e08c' - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/apache/hive/transfers/vertica_to_hive.py b/airflow/providers/apache/hive/transfers/vertica_to_hive.py index 725dbfc1282d5..a235ba3bf5011 100644 --- a/airflow/providers/apache/hive/transfers/vertica_to_hive.py +++ b/airflow/providers/apache/hive/transfers/vertica_to_hive.py @@ -26,7 +26,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.hive.hooks.hive import HiveCliHook from airflow.providers.vertica.hooks.vertica import VerticaHook -from airflow.utils.decorators import apply_defaults class VerticaToHiveOperator(BaseOperator): @@ -60,8 +59,9 @@ class VerticaToHiveOperator(BaseOperator): :type delimiter: str :param vertica_conn_id: source Vertica connection :type vertica_conn_id: str - :param hive_conn_id: destination hive connection - :type hive_conn_id: str + :param hive_cli_conn_id: Reference to the + :ref:`Hive CLI connection id `. + :type hive_cli_conn_id: str """ @@ -69,7 +69,6 @@ class VerticaToHiveOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#b4e0ff' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/kylin/operators/kylin_cube.py b/airflow/providers/apache/kylin/operators/kylin_cube.py index 20c107e6039e0..f5730d8a09c5b 100644 --- a/airflow/providers/apache/kylin/operators/kylin_cube.py +++ b/airflow/providers/apache/kylin/operators/kylin_cube.py @@ -25,7 +25,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.apache.kylin.hooks.kylin import KylinHook -from airflow.utils.decorators import apply_defaults class KylinCubeOperator(BaseOperator): @@ -110,7 +109,6 @@ class KylinCubeOperator(BaseOperator): jobs_end_status = {"FINISHED", "ERROR", "DISCARDED", "KILLED", "SUICIDAL", "STOPPED"} # pylint: disable=too-many-arguments,inconsistent-return-statements - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/livy/operators/livy.py b/airflow/providers/apache/livy/operators/livy.py index d13519431e2fd..6b06754f2b3d3 100644 --- a/airflow/providers/apache/livy/operators/livy.py +++ b/airflow/providers/apache/livy/operators/livy.py @@ -22,7 +22,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.apache.livy.hooks.livy import BatchState, LivyHook -from airflow.utils.decorators import apply_defaults class LivyOperator(BaseOperator): @@ -72,7 +71,6 @@ class LivyOperator(BaseOperator): template_fields = ('spark_params',) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/livy/sensors/livy.py b/airflow/providers/apache/livy/sensors/livy.py index 782ae089862de..0e7a67866ce15 100644 --- a/airflow/providers/apache/livy/sensors/livy.py +++ b/airflow/providers/apache/livy/sensors/livy.py @@ -20,7 +20,6 @@ from airflow.providers.apache.livy.hooks.livy import LivyHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class LivySensor(BaseSensorOperator): @@ -37,7 +36,6 @@ class LivySensor(BaseSensorOperator): template_fields = ('batch_id',) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/pig/operators/pig.py b/airflow/providers/apache/pig/operators/pig.py index 6d0f74e4fcfef..9c385fde28bdd 100644 --- a/airflow/providers/apache/pig/operators/pig.py +++ b/airflow/providers/apache/pig/operators/pig.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.pig.hooks.pig import PigCliHook -from airflow.utils.decorators import apply_defaults class PigOperator(BaseOperator): @@ -48,7 +47,6 @@ class PigOperator(BaseOperator): ) ui_color = '#f0e4ec' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/spark/operators/spark_jdbc.py b/airflow/providers/apache/spark/operators/spark_jdbc.py index 01dd28d3e2540..b15ffca1429f5 100644 --- a/airflow/providers/apache/spark/operators/spark_jdbc.py +++ b/airflow/providers/apache/spark/operators/spark_jdbc.py @@ -20,7 +20,6 @@ from airflow.providers.apache.spark.hooks.spark_jdbc import SparkJDBCHook from airflow.providers.apache.spark.operators.spark_submit import SparkSubmitOperator -from airflow.utils.decorators import apply_defaults # pylint: disable=too-many-instance-attributes @@ -120,7 +119,6 @@ class SparkJDBCOperator(SparkSubmitOperator): """ # pylint: disable=too-many-arguments,too-many-locals - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/spark/operators/spark_sql.py b/airflow/providers/apache/spark/operators/spark_sql.py index 69254092a2c9b..6c52fa2d58c03 100644 --- a/airflow/providers/apache/spark/operators/spark_sql.py +++ b/airflow/providers/apache/spark/operators/spark_sql.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.spark.hooks.spark_sql import SparkSqlHook -from airflow.utils.decorators import apply_defaults class SparkSqlOperator(BaseOperator): @@ -63,7 +62,6 @@ class SparkSqlOperator(BaseOperator): template_ext = [".sql", ".hql"] # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/spark/operators/spark_submit.py b/airflow/providers/apache/spark/operators/spark_submit.py index 16ca82e89664b..090f1046e0d4d 100644 --- a/airflow/providers/apache/spark/operators/spark_submit.py +++ b/airflow/providers/apache/spark/operators/spark_submit.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.spark.hooks.spark_submit import SparkSubmitHook from airflow.settings import WEB_COLORS -from airflow.utils.decorators import apply_defaults # pylint: disable=too-many-instance-attributes @@ -115,7 +114,6 @@ class SparkSubmitOperator(BaseOperator): ui_color = WEB_COLORS['LIGHTORANGE'] # pylint: disable=too-many-arguments,too-many-locals - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/apache/sqoop/operators/sqoop.py b/airflow/providers/apache/sqoop/operators/sqoop.py index b35797a6c2f1c..a790e4995cdd0 100644 --- a/airflow/providers/apache/sqoop/operators/sqoop.py +++ b/airflow/providers/apache/sqoop/operators/sqoop.py @@ -24,7 +24,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.apache.sqoop.hooks.sqoop import SqoopHook -from airflow.utils.decorators import apply_defaults # pylint: disable=too-many-instance-attributes @@ -110,7 +109,6 @@ class SqoopOperator(BaseOperator): ui_color = '#7D8CA4' # pylint: disable=too-many-arguments,too-many-locals - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/celery/sensors/celery_queue.py b/airflow/providers/celery/sensors/celery_queue.py index 4aa6b897d70b4..8808dcab95e03 100644 --- a/airflow/providers/celery/sensors/celery_queue.py +++ b/airflow/providers/celery/sensors/celery_queue.py @@ -21,7 +21,6 @@ from celery.app import control from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class CeleryQueueSensor(BaseSensorOperator): @@ -36,7 +35,6 @@ class CeleryQueueSensor(BaseSensorOperator): :type target_task_id: str """ - @apply_defaults def __init__(self, *, celery_queue: str, target_task_id: Optional[str] = None, **kwargs) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py index 0ddfe1acbcb93..5ab2040a1289f 100644 --- a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +++ b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py @@ -45,7 +45,6 @@ ) from airflow.providers.cncf.kubernetes.backcompat.pod_runtime_info_env import PodRuntimeInfoEnv from airflow.providers.cncf.kubernetes.utils import pod_launcher, xcom_sidecar -from airflow.utils.decorators import apply_defaults from airflow.utils.helpers import validate_key from airflow.utils.state import State from airflow.version import version as airflow_version @@ -174,7 +173,6 @@ class KubernetesPodOperator(BaseOperator): # pylint: disable=too-many-instance- ) # fmt: off - @apply_defaults def __init__( # pylint: disable=too-many-arguments,too-many-locals # fmt: on self, diff --git a/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py b/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py index a6cd381c27971..1e1fa5fbd698f 100644 --- a/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +++ b/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook -from airflow.utils.decorators import apply_defaults class SparkKubernetesOperator(BaseOperator): @@ -47,7 +46,6 @@ class SparkKubernetesOperator(BaseOperator): template_ext = ('yaml', 'yml', 'json') ui_color = '#f4a460' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py b/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py index 7a5efa2ab9dc8..da29e7974f298 100644 --- a/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py +++ b/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py @@ -22,7 +22,6 @@ from airflow.exceptions import AirflowException from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class SparkKubernetesSensor(BaseSensorOperator): @@ -52,7 +51,6 @@ class SparkKubernetesSensor(BaseSensorOperator): FAILURE_STATES = ("FAILED", "UNKNOWN") SUCCESS_STATES = ("COMPLETED",) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/databricks/operators/databricks.py b/airflow/providers/databricks/operators/databricks.py index 92d38afef17d3..ed37f753386ea 100644 --- a/airflow/providers/databricks/operators/databricks.py +++ b/airflow/providers/databricks/operators/databricks.py @@ -24,7 +24,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.databricks.hooks.databricks import DatabricksHook -from airflow.utils.decorators import apply_defaults XCOM_RUN_ID_KEY = 'run_id' XCOM_RUN_PAGE_URL_KEY = 'run_page_url' @@ -249,7 +248,6 @@ class DatabricksSubmitRunOperator(BaseOperator): ui_fgcolor = '#fff' # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, @@ -460,7 +458,6 @@ class DatabricksRunNowOperator(BaseOperator): ui_fgcolor = '#fff' # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/datadog/sensors/datadog.py b/airflow/providers/datadog/sensors/datadog.py index 24c2531d92e47..77fb2e701987e 100644 --- a/airflow/providers/datadog/sensors/datadog.py +++ b/airflow/providers/datadog/sensors/datadog.py @@ -22,7 +22,6 @@ from airflow.exceptions import AirflowException from airflow.providers.datadog.hooks.datadog import DatadogHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class DatadogSensor(BaseSensorOperator): @@ -55,7 +54,6 @@ class DatadogSensor(BaseSensorOperator): ui_color = '#66c3dd' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/dingding/operators/dingding.py b/airflow/providers/dingding/operators/dingding.py index 6be1bd3ad0040..3657ab8988cb9 100644 --- a/airflow/providers/dingding/operators/dingding.py +++ b/airflow/providers/dingding/operators/dingding.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.dingding.hooks.dingding import DingdingHook -from airflow.utils.decorators import apply_defaults class DingdingOperator(BaseOperator): @@ -47,7 +46,6 @@ class DingdingOperator(BaseOperator): template_fields = ('message',) ui_color = '#4ea4d4' # Dingding icon color - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/discord/operators/discord_webhook.py b/airflow/providers/discord/operators/discord_webhook.py index 2c1dc64174ad8..ebb5bc91f5848 100644 --- a/airflow/providers/discord/operators/discord_webhook.py +++ b/airflow/providers/discord/operators/discord_webhook.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.providers.discord.hooks.discord_webhook import DiscordWebhookHook from airflow.providers.http.operators.http import SimpleHttpOperator -from airflow.utils.decorators import apply_defaults class DiscordWebhookOperator(SimpleHttpOperator): @@ -56,7 +55,6 @@ class DiscordWebhookOperator(SimpleHttpOperator): template_fields = ['username', 'message'] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/docker/operators/docker.py b/airflow/providers/docker/operators/docker.py index 082206c00a93f..31093cdbd9792 100644 --- a/airflow/providers/docker/operators/docker.py +++ b/airflow/providers/docker/operators/docker.py @@ -25,7 +25,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.docker.hooks.docker import DockerHook -from airflow.utils.decorators import apply_defaults # pylint: disable=too-many-instance-attributes @@ -136,7 +135,6 @@ class DockerOperator(BaseOperator): ) # pylint: disable=too-many-arguments,too-many-locals - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index 0a811a4fe789b..ca315ad38e68b 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -22,7 +22,6 @@ from airflow.exceptions import AirflowException from airflow.providers.docker.operators.docker import DockerOperator -from airflow.utils.decorators import apply_defaults from airflow.utils.strings import get_random_string @@ -96,7 +95,6 @@ class DockerSwarmOperator(DockerOperator): :type enable_logging: bool """ - @apply_defaults def __init__(self, *, image: str, enable_logging: bool = True, **kwargs) -> None: super().__init__(image=image, **kwargs) diff --git a/airflow/providers/exasol/operators/exasol.py b/airflow/providers/exasol/operators/exasol.py index e7948bb228809..09e5cb8f17928 100644 --- a/airflow/providers/exasol/operators/exasol.py +++ b/airflow/providers/exasol/operators/exasol.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.exasol.hooks.exasol import ExasolHook -from airflow.utils.decorators import apply_defaults class ExasolOperator(BaseOperator): @@ -45,7 +44,6 @@ class ExasolOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#ededed' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/ftp/sensors/ftp.py b/airflow/providers/ftp/sensors/ftp.py index 49bb20d2cd489..1ea6eaf215421 100644 --- a/airflow/providers/ftp/sensors/ftp.py +++ b/airflow/providers/ftp/sensors/ftp.py @@ -20,7 +20,6 @@ from airflow.providers.ftp.hooks.ftp import FTPHook, FTPSHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class FTPSensor(BaseSensorOperator): @@ -44,7 +43,6 @@ class FTPSensor(BaseSensorOperator): error_code_pattern = re.compile(r"([\d]+)") - @apply_defaults def __init__( self, *, path: str, ftp_conn_id: str = 'ftp_default', fail_on_transient_errors: bool = True, **kwargs ) -> None: diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 6044c031d0ce4..4d2f5a19fbb22 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -25,6 +25,12 @@ Changelog Breaking changes ~~~~~~~~~~~~~~~~ +Change in ``AutoMLPredictOperator`` +``````````````````````````````````` + +The ``params`` parameter in :class:`~airflow.providers.google.cloud.operators.automl.AutoMLPredictOperator` class +was renamed ``operation_params`` because it conflicted with a ``param`` parameter in the ``BaseOperator`` class. + Integration with the ``apache.beam`` provider ````````````````````````````````````````````` diff --git a/airflow/providers/google/ads/example_dags/example_ads.py b/airflow/providers/google/ads/example_dags/example_ads.py index e38fd473aae04..959ce05c6ee72 100644 --- a/airflow/providers/google/ads/example_dags/example_ads.py +++ b/airflow/providers/google/ads/example_dags/example_ads.py @@ -27,7 +27,7 @@ # [START howto_google_ads_env_variables] CLIENT_IDS = ["1111111111", "2222222222"] -BUCKET = os.environ.get("GOOGLE_ADS_BUCKET", "gs://test-google-ads-bucket") +BUCKET = os.environ.get("GOOGLE_ADS_BUCKET", "gs://INVALID BUCKET NAME") GCS_OBJ_PATH = "folder_name/google-ads-api-results.csv" GCS_ACCOUNTS_CSV = "folder_name/accounts.csv" QUERY = """ diff --git a/airflow/providers/google/ads/operators/ads.py b/airflow/providers/google/ads/operators/ads.py index 341f7306b4605..86aa03ca52bbd 100644 --- a/airflow/providers/google/ads/operators/ads.py +++ b/airflow/providers/google/ads/operators/ads.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.ads.hooks.ads import GoogleAdsHook from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults class GoogleAdsListAccountsOperator(BaseOperator): @@ -70,7 +69,6 @@ class GoogleAdsListAccountsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/ads/transfers/ads_to_gcs.py b/airflow/providers/google/ads/transfers/ads_to_gcs.py index 9713a33b5c6b0..d8ea1b67f589a 100644 --- a/airflow/providers/google/ads/transfers/ads_to_gcs.py +++ b/airflow/providers/google/ads/transfers/ads_to_gcs.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.ads.hooks.ads import GoogleAdsHook from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults class GoogleAdsToGcsOperator(BaseOperator): @@ -78,7 +77,6 @@ class GoogleAdsToGcsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py index 1560dda31dbb6..45207352b7138 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py @@ -34,7 +34,7 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_TEXT_CLS_BUCKET = os.environ.get("GCP_AUTOML_TEXT_CLS_BUCKET", "gs://") +GCP_AUTOML_TEXT_CLS_BUCKET = os.environ.get("GCP_AUTOML_TEXT_CLS_BUCKET", "gs://INVALID BUCKET NAME") # Example values DATASET_ID = "" diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_extraction.py b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_extraction.py index ac63c02ccc09d..6e28fc7daab78 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_extraction.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_extraction.py @@ -34,7 +34,9 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_TEXT_BUCKET = os.environ.get("GCP_AUTOML_TEXT_BUCKET", "gs://cloud-ml-data/NL-entity/dataset.csv") +GCP_AUTOML_TEXT_BUCKET = os.environ.get( + "GCP_AUTOML_TEXT_BUCKET", "gs://INVALID BUCKET NAME/NL-entity/dataset.csv" +) # Example values DATASET_ID = "" diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py index 8645f0a0f563b..00e49cf54e253 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py @@ -34,7 +34,7 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_SENTIMENT_BUCKET = os.environ.get("GCP_AUTOML_SENTIMENT_BUCKET", "gs://") +GCP_AUTOML_SENTIMENT_BUCKET = os.environ.get("GCP_AUTOML_SENTIMENT_BUCKET", "gs://INVALID BUCKET NAME") # Example values DATASET_ID = "" diff --git a/airflow/providers/google/cloud/example_dags/example_automl_tables.py b/airflow/providers/google/cloud/example_dags/example_automl_tables.py index 117bd34c3e8db..4f3555282f719 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_tables.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_tables.py @@ -45,7 +45,7 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") GCP_AUTOML_DATASET_BUCKET = os.environ.get( - "GCP_AUTOML_DATASET_BUCKET", "gs://cloud-ml-tables-data/bank-marketing.csv" + "GCP_AUTOML_DATASET_BUCKET", "gs://INVALID BUCKET NAME/bank-marketing.csv" ) TARGET = os.environ.get("GCP_AUTOML_TARGET", "Deposit") diff --git a/airflow/providers/google/cloud/example_dags/example_automl_translation.py b/airflow/providers/google/cloud/example_dags/example_automl_translation.py index 97bf143ee4a3b..c0216925525fd 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_translation.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_translation.py @@ -34,7 +34,9 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_TRANSLATION_BUCKET = os.environ.get("GCP_AUTOML_TRANSLATION_BUCKET", "gs://project-vcm/file") +GCP_AUTOML_TRANSLATION_BUCKET = os.environ.get( + "GCP_AUTOML_TRANSLATION_BUCKET", "gs://INVALID BUCKET NAME/file" +) # Example values DATASET_ID = "TRL123456789" diff --git a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py b/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py index 1605046f19b06..140ce0ee9193a 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py @@ -35,7 +35,7 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") GCP_AUTOML_VIDEO_BUCKET = os.environ.get( - "GCP_AUTOML_VIDEO_BUCKET", "gs://automl-video-demo-data/hmdb_split1.csv" + "GCP_AUTOML_VIDEO_BUCKET", "gs://INVALID BUCKET NAME/hmdb_split1.csv" ) # Example values diff --git a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py b/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py index 68d3ebd681a26..f1984129950d3 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py @@ -36,7 +36,7 @@ GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") GCP_AUTOML_TRACKING_BUCKET = os.environ.get( "GCP_AUTOML_TRACKING_BUCKET", - "gs://automl-video-datasets/youtube_8m_videos_animal_tiny.csv", + "gs://INVALID BUCKET NAME/youtube_8m_videos_animal_tiny.csv", ) # Example values diff --git a/airflow/providers/google/cloud/example_dags/example_automl_vision_classification.py b/airflow/providers/google/cloud/example_dags/example_automl_vision_classification.py index 2d16db214852d..ed96adc754a0c 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_vision_classification.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_vision_classification.py @@ -34,7 +34,7 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_VISION_BUCKET = os.environ.get("GCP_AUTOML_VISION_BUCKET", "gs://your-bucket") +GCP_AUTOML_VISION_BUCKET = os.environ.get("GCP_AUTOML_VISION_BUCKET", "gs://INVALID BUCKET NAME") # Example values DATASET_ID = "ICN123455678" diff --git a/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py b/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py index 3be9eba969749..f883c375f24f6 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py @@ -35,7 +35,7 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") GCP_AUTOML_DETECTION_BUCKET = os.environ.get( - "GCP_AUTOML_DETECTION_BUCKET", "gs://cloud-ml-data/img/openimage/csv/salads_ml_use.csv" + "GCP_AUTOML_DETECTION_BUCKET", "gs://INVALID BUCKET NAME/img/openimage/csv/salads_ml_use.csv" ) # Example values diff --git a/airflow/providers/google/cloud/example_dags/example_azure_fileshare_to_gcs.py b/airflow/providers/google/cloud/example_dags/example_azure_fileshare_to_gcs.py index c260cb8616c0d..bf28fe8aaaa19 100644 --- a/airflow/providers/google/cloud/example_dags/example_azure_fileshare_to_gcs.py +++ b/airflow/providers/google/cloud/example_dags/example_azure_fileshare_to_gcs.py @@ -20,7 +20,7 @@ from airflow import DAG from airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs import AzureFileShareToGCSOperator -DEST_GCS_BUCKET = os.environ.get('GCP_GCS_BUCKET', 'gs://test-gcs-example-bucket') +DEST_GCS_BUCKET = os.environ.get('GCP_GCS_BUCKET', 'gs://INVALID BUCKET NAME') AZURE_SHARE_NAME = os.environ.get('AZURE_SHARE_NAME', 'test-azure-share') AZURE_DIRECTORY_NAME = "test-azure-dir" diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_dts.py b/airflow/providers/google/cloud/example_dags/example_bigquery_dts.py index da13c9da703d1..b0e2b86a9f141 100644 --- a/airflow/providers/google/cloud/example_dags/example_bigquery_dts.py +++ b/airflow/providers/google/cloud/example_dags/example_bigquery_dts.py @@ -32,7 +32,7 @@ from airflow.utils.dates import days_ago GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project") -BUCKET_URI = os.environ.get("GCP_DTS_BUCKET_URI", "gs://cloud-ml-tables-data/bank-marketing.csv") +BUCKET_URI = os.environ.get("GCP_DTS_BUCKET_URI", "gs://INVALID BUCKET NAME/bank-marketing.csv") GCP_DTS_BQ_DATASET = os.environ.get("GCP_DTS_BQ_DATASET", "test_dts") GCP_DTS_BQ_TABLE = os.environ.get("GCP_DTS_BQ_TABLE", "GCS_Test") diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py b/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py index 35ee4d821d0e9..a72ff36f58c5f 100644 --- a/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py +++ b/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py @@ -36,6 +36,7 @@ BigQueryPatchDatasetOperator, BigQueryUpdateDatasetOperator, BigQueryUpdateTableOperator, + BigQueryUpdateTableSchemaOperator, BigQueryUpsertTableOperator, ) from airflow.utils.dates import days_ago @@ -47,7 +48,7 @@ LOCATION_DATASET_NAME = f"{DATASET_NAME}_location" DATA_SAMPLE_GCS_URL = os.environ.get( "GCP_BIGQUERY_DATA_GCS_URL", - "gs://cloud-samples-data/bigquery/us-states/us-states.csv", + "gs://INVALID BUCKET NAME/bigquery/us-states/us-states.csv", ) DATA_SAMPLE_GCS_URL_PARTS = urlparse(DATA_SAMPLE_GCS_URL) @@ -73,6 +74,18 @@ ) # [END howto_operator_bigquery_create_table] + # [START howto_operator_bigquery_update_table_schema] + update_table_schema = BigQueryUpdateTableSchemaOperator( + task_id="update_table_schema", + dataset_id=DATASET_NAME, + table_id="test_table", + schema_fields_updates=[ + {"name": "emp_name", "description": "Name of employee"}, + {"name": "salary", "description": "Monthly salary in USD"}, + ], + ) + # [END howto_operator_bigquery_update_table_schema] + # [START howto_operator_bigquery_delete_table] delete_table = BigQueryDeleteTableOperator( task_id="delete_table", @@ -216,6 +229,7 @@ delete_view, ] >> upsert_table + >> update_table_schema >> delete_materialized_view >> delete_table >> delete_dataset diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_to_gcs.py b/airflow/providers/google/cloud/example_dags/example_bigquery_to_gcs.py index 70841573b73d9..4e6de6c0cc1b0 100644 --- a/airflow/providers/google/cloud/example_dags/example_bigquery_to_gcs.py +++ b/airflow/providers/google/cloud/example_dags/example_bigquery_to_gcs.py @@ -32,7 +32,7 @@ PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project") DATASET_NAME = os.environ.get("GCP_BIGQUERY_DATASET_NAME", "test_dataset_transfer") -DATA_EXPORT_BUCKET_NAME = os.environ.get("GCP_BIGQUERY_EXPORT_BUCKET_NAME", "test-bigquery-gcs-data") +DATA_EXPORT_BUCKET_NAME = os.environ.get("GCP_BIGQUERY_EXPORT_BUCKET_NAME", "INVALID BUCKET NAME") TABLE = "table_42" with models.DAG( diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_transfer.py b/airflow/providers/google/cloud/example_dags/example_bigquery_transfer.py index 82a06a2723f52..658d8a09697b1 100644 --- a/airflow/providers/google/cloud/example_dags/example_bigquery_transfer.py +++ b/airflow/providers/google/cloud/example_dags/example_bigquery_transfer.py @@ -33,7 +33,7 @@ PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project") DATASET_NAME = os.environ.get("GCP_BIGQUERY_DATASET_NAME", "test_dataset_transfer") -DATA_EXPORT_BUCKET_NAME = os.environ.get("GCP_BIGQUERY_EXPORT_BUCKET_NAME", "test-bigquery-sample-data") +DATA_EXPORT_BUCKET_NAME = os.environ.get("GCP_BIGQUERY_EXPORT_BUCKET_NAME", "INVALID BUCKET NAME") ORIGIN = "origin" TARGET = "target" diff --git a/airflow/providers/google/cloud/example_dags/example_cloud_build.py b/airflow/providers/google/cloud/example_dags/example_cloud_build.py index 9a74039296f51..b62f90fdf3e62 100644 --- a/airflow/providers/google/cloud/example_dags/example_cloud_build.py +++ b/airflow/providers/google/cloud/example_dags/example_cloud_build.py @@ -40,7 +40,7 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project") -GCP_SOURCE_ARCHIVE_URL = os.environ.get("GCP_CLOUD_BUILD_ARCHIVE_URL", "gs://example-bucket/file") +GCP_SOURCE_ARCHIVE_URL = os.environ.get("GCP_CLOUD_BUILD_ARCHIVE_URL", "gs://INVALID BUCKET NAME/file") GCP_SOURCE_REPOSITORY_NAME = os.environ.get("GCP_CLOUD_BUILD_REPOSITORY_NAME", "repository-name") GCP_SOURCE_ARCHIVE_URL_PARTS = urlparse(GCP_SOURCE_ARCHIVE_URL) diff --git a/airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py b/airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py index 2a88c5dcadb0e..b7315ed65f269 100644 --- a/airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py +++ b/airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py @@ -63,7 +63,7 @@ "GCP_MEMORYSTORE_MEMCACHED_INSTANCE_NAME", "test-memorystore-memcached-1" ) -BUCKET_NAME = os.environ.get("GCP_MEMORYSTORE_BUCKET", "test-memorystore-bucket") +BUCKET_NAME = os.environ.get("GCP_MEMORYSTORE_BUCKET", "INVALID BUCKET NAME") EXPORT_GCS_URL = f"gs://{BUCKET_NAME}/my-export.rdb" # [START howto_operator_instance] diff --git a/airflow/providers/google/cloud/example_dags/example_cloud_sql.py b/airflow/providers/google/cloud/example_dags/example_cloud_sql.py index 240639ffcb9f2..286ecf38f0799 100644 --- a/airflow/providers/google/cloud/example_dags/example_cloud_sql.py +++ b/airflow/providers/google/cloud/example_dags/example_cloud_sql.py @@ -52,8 +52,8 @@ INSTANCE_NAME2 = os.environ.get('GCSQL_MYSQL_INSTANCE_NAME2', 'test-mysql2') DB_NAME = os.environ.get('GCSQL_MYSQL_DATABASE_NAME', 'testdb') -EXPORT_URI = os.environ.get('GCSQL_MYSQL_EXPORT_URI', 'gs://bucketName/fileName') -IMPORT_URI = os.environ.get('GCSQL_MYSQL_IMPORT_URI', 'gs://bucketName/fileName') +EXPORT_URI = os.environ.get('GCSQL_MYSQL_EXPORT_URI', 'gs://INVALID BUCKET NAME/fileName') +IMPORT_URI = os.environ.get('GCSQL_MYSQL_IMPORT_URI', 'gs://INVALID BUCKET NAME/fileName') # Bodies below represent Cloud SQL instance resources: # https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances diff --git a/airflow/providers/google/cloud/example_dags/example_datacatalog.py b/airflow/providers/google/cloud/example_dags/example_datacatalog.py index 7f4544b5e5abc..e6e94f2913925 100644 --- a/airflow/providers/google/cloud/example_dags/example_datacatalog.py +++ b/airflow/providers/google/cloud/example_dags/example_datacatalog.py @@ -92,7 +92,7 @@ entry={ "display_name": "Wizard", "type_": "FILESET", - "gcs_fileset_spec": {"file_patterns": ["gs://test-datacatalog/**"]}, + "gcs_fileset_spec": {"file_patterns": ["gs://INVALID BUCKET NAME/**"]}, }, ) # [END howto_operator_gcp_datacatalog_create_entry_gcs] diff --git a/airflow/providers/google/cloud/example_dags/example_dataflow.py b/airflow/providers/google/cloud/example_dags/example_dataflow.py index a6315bb5cd86f..6e58ff4d67ffe 100644 --- a/airflow/providers/google/cloud/example_dags/example_dataflow.py +++ b/airflow/providers/google/cloud/example_dags/example_dataflow.py @@ -41,11 +41,11 @@ from airflow.providers.google.cloud.transfers.gcs_to_local import GCSToLocalFilesystemOperator from airflow.utils.dates import days_ago -GCS_TMP = os.environ.get('GCP_DATAFLOW_GCS_TMP', 'gs://test-dataflow-example/temp/') -GCS_STAGING = os.environ.get('GCP_DATAFLOW_GCS_STAGING', 'gs://test-dataflow-example/staging/') -GCS_OUTPUT = os.environ.get('GCP_DATAFLOW_GCS_OUTPUT', 'gs://test-dataflow-example/output') -GCS_JAR = os.environ.get('GCP_DATAFLOW_JAR', 'gs://test-dataflow-example/word-count-beam-bundled-0.1.jar') -GCS_PYTHON = os.environ.get('GCP_DATAFLOW_PYTHON', 'gs://test-dataflow-example/wordcount_debugging.py') +GCS_TMP = os.environ.get('GCP_DATAFLOW_GCS_TMP', 'gs://INVALID BUCKET NAME/temp/') +GCS_STAGING = os.environ.get('GCP_DATAFLOW_GCS_STAGING', 'gs://INVALID BUCKET NAME/staging/') +GCS_OUTPUT = os.environ.get('GCP_DATAFLOW_GCS_OUTPUT', 'gs://INVALID BUCKET NAME/output') +GCS_JAR = os.environ.get('GCP_DATAFLOW_JAR', 'gs://INVALID BUCKET NAME/word-count-beam-bundled-0.1.jar') +GCS_PYTHON = os.environ.get('GCP_DATAFLOW_PYTHON', 'gs://INVALID BUCKET NAME/wordcount_debugging.py') GCS_JAR_PARTS = urlparse(GCS_JAR) GCS_JAR_BUCKET_NAME = GCS_JAR_PARTS.netloc diff --git a/airflow/providers/google/cloud/example_dags/example_dataflow_flex_template.py b/airflow/providers/google/cloud/example_dags/example_dataflow_flex_template.py index ce002c73c5950..d67550ceddfaf 100644 --- a/airflow/providers/google/cloud/example_dags/example_dataflow_flex_template.py +++ b/airflow/providers/google/cloud/example_dags/example_dataflow_flex_template.py @@ -38,7 +38,7 @@ PUBSUB_FLEX_TEMPLATE_SUBSCRIPTION = PUBSUB_FLEX_TEMPLATE_TOPIC GCS_FLEX_TEMPLATE_TEMPLATE_PATH = os.environ.get( 'GCP_DATAFLOW_GCS_FLEX_TEMPLATE_TEMPLATE_PATH', - "gs://test-airflow-dataflow-flex-template/samples/dataflow/templates/streaming-beam-sql.json", + "gs://INVALID BUCKET NAME/samples/dataflow/templates/streaming-beam-sql.json", ) BQ_FLEX_TEMPLATE_DATASET = os.environ.get('GCP_DATAFLOW_BQ_FLEX_TEMPLATE_DATASET', 'airflow_dataflow_samples') BQ_FLEX_TEMPLATE_LOCATION = os.environ.get('GCP_DATAFLOW_BQ_FLEX_TEMPLATE_LOCATION>', 'us-west1') diff --git a/airflow/providers/google/cloud/example_dags/example_dataprep.py b/airflow/providers/google/cloud/example_dags/example_dataprep.py index e24f161e68fe8..b3b654007f06f 100644 --- a/airflow/providers/google/cloud/example_dags/example_dataprep.py +++ b/airflow/providers/google/cloud/example_dags/example_dataprep.py @@ -29,7 +29,7 @@ DATAPREP_JOB_ID = int(os.environ.get('DATAPREP_JOB_ID', 12345677)) DATAPREP_JOB_RECIPE_ID = int(os.environ.get('DATAPREP_JOB_RECIPE_ID', 12345677)) -DATAPREP_BUCKET = os.environ.get("DATAPREP_BUCKET", "gs://afl-sql/name@email.com") +DATAPREP_BUCKET = os.environ.get("DATAPREP_BUCKET", "gs://INVALID BUCKET NAME/name@email.com") DATA = { "wrangledDataset": {"id": DATAPREP_JOB_RECIPE_ID}, diff --git a/airflow/providers/google/cloud/example_dags/example_dlp.py b/airflow/providers/google/cloud/example_dags/example_dlp.py index 43a3d92352a3d..9a056d183a024 100644 --- a/airflow/providers/google/cloud/example_dags/example_dlp.py +++ b/airflow/providers/google/cloud/example_dags/example_dlp.py @@ -53,7 +53,7 @@ ) INSPECT_CONFIG = InspectConfig(info_types=[{"name": "PHONE_NUMBER"}, {"name": "US_TOLLFREE_PHONE_NUMBER"}]) INSPECT_TEMPLATE = InspectTemplate(inspect_config=INSPECT_CONFIG) -OUTPUT_BUCKET = os.environ.get("DLP_OUTPUT_BUCKET", "gs://test-dlp-airflow") +OUTPUT_BUCKET = os.environ.get("DLP_OUTPUT_BUCKET", "gs://INVALID BUCKET NAME") OUTPUT_FILENAME = "test.txt" OBJECT_GCS_URI = os.path.join(OUTPUT_BUCKET, "tmp") diff --git a/airflow/providers/google/cloud/example_dags/example_life_sciences.py b/airflow/providers/google/cloud/example_dags/example_life_sciences.py index c05e6c4d96d75..1bd035dabd296 100644 --- a/airflow/providers/google/cloud/example_dags/example_life_sciences.py +++ b/airflow/providers/google/cloud/example_dags/example_life_sciences.py @@ -23,7 +23,7 @@ from airflow.utils import dates PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project-id") -BUCKET = os.environ.get("GCP_GCS_LIFE_SCIENCES_BUCKET", "example-life-sciences-bucket") +BUCKET = os.environ.get("GCP_GCS_LIFE_SCIENCES_BUCKET", "INVALID BUCKET NAME") FILENAME = os.environ.get("GCP_GCS_LIFE_SCIENCES_FILENAME", 'input.in') LOCATION = os.environ.get("GCP_LIFE_SCIENCES_LOCATION", 'us-central1') diff --git a/airflow/providers/google/cloud/example_dags/example_mlengine.py b/airflow/providers/google/cloud/example_dags/example_mlengine.py index 43a0e469d85a1..d82d166b5feeb 100644 --- a/airflow/providers/google/cloud/example_dags/example_mlengine.py +++ b/airflow/providers/google/cloud/example_dags/example_mlengine.py @@ -42,19 +42,19 @@ MODEL_NAME = os.environ.get("GCP_MLENGINE_MODEL_NAME", "model_name") -SAVED_MODEL_PATH = os.environ.get("GCP_MLENGINE_SAVED_MODEL_PATH", "gs://test-airflow-mlengine/saved-model/") -JOB_DIR = os.environ.get("GCP_MLENGINE_JOB_DIR", "gs://test-airflow-mlengine/keras-job-dir") +SAVED_MODEL_PATH = os.environ.get("GCP_MLENGINE_SAVED_MODEL_PATH", "gs://INVALID BUCKET NAME/saved-model/") +JOB_DIR = os.environ.get("GCP_MLENGINE_JOB_DIR", "gs://INVALID BUCKET NAME/keras-job-dir") PREDICTION_INPUT = os.environ.get( - "GCP_MLENGINE_PREDICTION_INPUT", "gs://test-airflow-mlengine/prediction_input.json" + "GCP_MLENGINE_PREDICTION_INPUT", "gs://INVALID BUCKET NAME/prediction_input.json" ) PREDICTION_OUTPUT = os.environ.get( - "GCP_MLENGINE_PREDICTION_OUTPUT", "gs://test-airflow-mlengine/prediction_output" + "GCP_MLENGINE_PREDICTION_OUTPUT", "gs://INVALID BUCKET NAME/prediction_output" ) -TRAINER_URI = os.environ.get("GCP_MLENGINE_TRAINER_URI", "gs://test-airflow-mlengine/trainer.tar.gz") +TRAINER_URI = os.environ.get("GCP_MLENGINE_TRAINER_URI", "gs://INVALID BUCKET NAME/trainer.tar.gz") TRAINER_PY_MODULE = os.environ.get("GCP_MLENGINE_TRAINER_TRAINER_PY_MODULE", "trainer.task") -SUMMARY_TMP = os.environ.get("GCP_MLENGINE_DATAFLOW_TMP", "gs://test-airflow-mlengine/tmp/") -SUMMARY_STAGING = os.environ.get("GCP_MLENGINE_DATAFLOW_STAGING", "gs://test-airflow-mlengine/staging/") +SUMMARY_TMP = os.environ.get("GCP_MLENGINE_DATAFLOW_TMP", "gs://INVALID BUCKET NAME/tmp/") +SUMMARY_STAGING = os.environ.get("GCP_MLENGINE_DATAFLOW_STAGING", "gs://INVALID BUCKET NAME/staging/") default_args = {"params": {"model_name": MODEL_NAME}} diff --git a/airflow/providers/google/cloud/example_dags/example_natural_language.py b/airflow/providers/google/cloud/example_dags/example_natural_language.py index 11a9246495ee6..9038052b9ae69 100644 --- a/airflow/providers/google/cloud/example_dags/example_natural_language.py +++ b/airflow/providers/google/cloud/example_dags/example_natural_language.py @@ -44,7 +44,7 @@ # [END howto_operator_gcp_natural_language_document_text] # [START howto_operator_gcp_natural_language_document_gcs] -GCS_CONTENT_URI = "gs://my-text-bucket/sentiment-me.txt" +GCS_CONTENT_URI = "gs://INVALID BUCKET NAME/sentiment-me.txt" document_gcs = Document(gcs_content_uri=GCS_CONTENT_URI, type="PLAIN_TEXT") # [END howto_operator_gcp_natural_language_document_gcs] diff --git a/airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py b/airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py index 19921a5c72dbe..677bd4c037de3 100644 --- a/airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py +++ b/airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py @@ -25,7 +25,7 @@ from airflow.utils.dates import days_ago PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project") -GCS_BUCKET = os.environ.get("GCP_GCS_BUCKET_NAME", "postgres_to_gcs_example") +GCS_BUCKET = os.environ.get("GCP_GCS_BUCKET_NAME", "INVALID BUCKET NAME") FILENAME = "test_file" SQL_QUERY = "select * from test_table;" diff --git a/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py b/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py index 3534b7631964f..759c429e65e7a 100644 --- a/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py +++ b/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py @@ -32,7 +32,7 @@ from airflow.utils.dates import days_ago GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", 'example-project') -GCS_BUCKET = os.environ.get("GCP_PRESTO_TO_GCS_BUCKET_NAME", "test-presto-to-gcs-bucket") +GCS_BUCKET = os.environ.get("GCP_PRESTO_TO_GCS_BUCKET_NAME", "INVALID BUCKET NAME") DATASET_NAME = os.environ.get("GCP_PRESTO_TO_GCS_DATASET_NAME", "test_presto_to_gcs_dataset") SOURCE_MULTIPLE_TYPES = "memory.default.test_multiple_types" diff --git a/airflow/providers/google/cloud/example_dags/example_speech_to_text.py b/airflow/providers/google/cloud/example_dags/example_speech_to_text.py index 66778c741a01b..f067f9d26b099 100644 --- a/airflow/providers/google/cloud/example_dags/example_speech_to_text.py +++ b/airflow/providers/google/cloud/example_dags/example_speech_to_text.py @@ -24,7 +24,7 @@ from airflow.utils import dates GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project") -BUCKET_NAME = os.environ.get("GCP_SPEECH_TO_TEXT_TEST_BUCKET", "gcp-speech-to-text-test-bucket") +BUCKET_NAME = os.environ.get("GCP_SPEECH_TO_TEXT_TEST_BUCKET", "INVALID BUCKET NAME") # [START howto_operator_speech_to_text_gcp_filename] FILENAME = "gcp-speech-test-file" diff --git a/airflow/providers/google/cloud/example_dags/example_translate_speech.py b/airflow/providers/google/cloud/example_dags/example_translate_speech.py index f8b47842ffa43..9548d18049289 100644 --- a/airflow/providers/google/cloud/example_dags/example_translate_speech.py +++ b/airflow/providers/google/cloud/example_dags/example_translate_speech.py @@ -24,7 +24,7 @@ from airflow.utils import dates GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project") -BUCKET_NAME = os.environ.get("GCP_TRANSLATE_SPEECH_TEST_BUCKET", "gcp-translate-speech-test-bucket") +BUCKET_NAME = os.environ.get("GCP_TRANSLATE_SPEECH_TEST_BUCKET", "INVALID BUCKET NAME") # [START howto_operator_translate_speech_gcp_filename] FILENAME = "gcp-speech-test-file" diff --git a/airflow/providers/google/cloud/example_dags/example_trino_to_gcs.py b/airflow/providers/google/cloud/example_dags/example_trino_to_gcs.py index 32dc8a004b79f..209c51e2c2d9c 100644 --- a/airflow/providers/google/cloud/example_dags/example_trino_to_gcs.py +++ b/airflow/providers/google/cloud/example_dags/example_trino_to_gcs.py @@ -32,7 +32,7 @@ from airflow.utils.dates import days_ago GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", 'example-project') -GCS_BUCKET = os.environ.get("GCP_TRINO_TO_GCS_BUCKET_NAME", "test-trino-to-gcs-bucket") +GCS_BUCKET = os.environ.get("GCP_TRINO_TO_GCS_BUCKET_NAME", "INVALID BUCKET NAME") DATASET_NAME = os.environ.get("GCP_TRINO_TO_GCS_DATASET_NAME", "test_trino_to_gcs_dataset") SOURCE_MULTIPLE_TYPES = "memory.default.test_multiple_types" diff --git a/airflow/providers/google/cloud/example_dags/example_video_intelligence.py b/airflow/providers/google/cloud/example_dags/example_video_intelligence.py index dbb63723d5921..67f98e5e14e86 100644 --- a/airflow/providers/google/cloud/example_dags/example_video_intelligence.py +++ b/airflow/providers/google/cloud/example_dags/example_video_intelligence.py @@ -38,7 +38,7 @@ from airflow.utils.dates import days_ago # [START howto_operator_video_intelligence_os_args] -GCP_BUCKET_NAME = os.environ.get("GCP_VIDEO_INTELLIGENCE_BUCKET_NAME", "test-bucket-name") +GCP_BUCKET_NAME = os.environ.get("GCP_VIDEO_INTELLIGENCE_BUCKET_NAME", "INVALID BUCKET NAME") # [END howto_operator_video_intelligence_os_args] diff --git a/airflow/providers/google/cloud/example_dags/example_vision.py b/airflow/providers/google/cloud/example_dags/example_vision.py index a191ad89da95b..516a70dfaf53d 100644 --- a/airflow/providers/google/cloud/example_dags/example_vision.py +++ b/airflow/providers/google/cloud/example_dags/example_vision.py @@ -83,8 +83,12 @@ GCP_VISION_PRODUCT_SET_ID = os.environ.get('GCP_VISION_PRODUCT_SET_ID', 'product_set_explicit_id') GCP_VISION_PRODUCT_ID = os.environ.get('GCP_VISION_PRODUCT_ID', 'product_explicit_id') GCP_VISION_REFERENCE_IMAGE_ID = os.environ.get('GCP_VISION_REFERENCE_IMAGE_ID', 'reference_image_explicit_id') -GCP_VISION_REFERENCE_IMAGE_URL = os.environ.get('GCP_VISION_REFERENCE_IMAGE_URL', 'gs://bucket/image1.jpg') -GCP_VISION_ANNOTATE_IMAGE_URL = os.environ.get('GCP_VISION_ANNOTATE_IMAGE_URL', 'gs://bucket/image2.jpg') +GCP_VISION_REFERENCE_IMAGE_URL = os.environ.get( + 'GCP_VISION_REFERENCE_IMAGE_URL', 'gs://INVALID BUCKET NAME/image1.jpg' +) +GCP_VISION_ANNOTATE_IMAGE_URL = os.environ.get( + 'GCP_VISION_ANNOTATE_IMAGE_URL', 'gs://INVALID BUCKET NAME/image2.jpg' +) # [START howto_operator_vision_product_set] product_set = ProductSet(display_name='My Product Set') diff --git a/airflow/providers/google/cloud/hooks/bigquery.py b/airflow/providers/google/cloud/hooks/bigquery.py index 044c014bf6377..e36baf548c9fd 100644 --- a/airflow/providers/google/cloud/hooks/bigquery.py +++ b/airflow/providers/google/cloud/hooks/bigquery.py @@ -1373,6 +1373,101 @@ def get_schema(self, dataset_id: str, table_id: str, project_id: Optional[str] = table = self.get_client(project_id=project_id).get_table(table_ref) return {"fields": [s.to_api_repr() for s in table.schema]} + @GoogleBaseHook.fallback_to_default_project_id + def update_table_schema( + self, + schema_fields_updates: List[Dict[str, Any]], + include_policy_tags: bool, + dataset_id: str, + table_id: str, + project_id: Optional[str] = None, + ) -> None: + """ + Update fields within a schema for a given dataset and table. Note that + some fields in schemas are immutable and trying to change them will cause + an exception. + If a new field is included it will be inserted which requires all required fields to be set. + See https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TableSchema + + :param include_policy_tags: If set to True policy tags will be included in + the update request which requires special permissions even if unchanged + see https://cloud.google.com/bigquery/docs/column-level-security#roles + :type include_policy_tags: bool + :param dataset_id: the dataset ID of the requested table to be updated + :type dataset_id: str + :param table_id: the table ID of the table to be updated + :type table_id: str + :param schema_fields_updates: a partial schema resource. see + https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TableSchema + + **Example**: :: + + schema_fields_updates=[ + {"name": "emp_name", "description": "Some New Description"}, + {"name": "salary", "description": "Some New Description"}, + {"name": "departments", "fields": [ + {"name": "name", "description": "Some New Description"}, + {"name": "type", "description": "Some New Description"} + ]}, + ] + + :type schema_fields_updates: List[dict] + :param project_id: The name of the project where we want to update the table. + :type project_id: str + """ + + def _build_new_schema( + current_schema: List[Dict[str, Any]], schema_fields_updates: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: + + # Turn schema_field_updates into a dict keyed on field names + schema_fields_updates = {field["name"]: field for field in deepcopy(schema_fields_updates)} + + # Create a new dict for storing the new schema, initated based on the current_schema + # as of Python 3.6, dicts retain order. + new_schema = {field["name"]: field for field in deepcopy(current_schema)} + + # Each item in schema_fields_updates contains a potential patch + # to a schema field, iterate over them + for field_name, patched_value in schema_fields_updates.items(): + # If this field already exists, update it + if field_name in new_schema: + # If this field is of type RECORD and has a fields key we need to patch it recursively + if "fields" in patched_value: + patched_value["fields"] = _build_new_schema( + new_schema[field_name]["fields"], patched_value["fields"] + ) + # Update the new_schema with the patched value + new_schema[field_name].update(patched_value) + # This is a new field, just include the whole configuration for it + else: + new_schema[field_name] = patched_value + + return list(new_schema.values()) + + def _remove_policy_tags(schema: List[Dict[str, Any]]): + for field in schema: + if "policyTags" in field: + del field["policyTags"] + if "fields" in field: + _remove_policy_tags(field["fields"]) + + current_table_schema = self.get_schema( + dataset_id=dataset_id, table_id=table_id, project_id=project_id + )["fields"] + new_schema = _build_new_schema(current_table_schema, schema_fields_updates) + + if not include_policy_tags: + _remove_policy_tags(new_schema) + + self.update_table( + table_resource={"schema": {"fields": new_schema}}, + fields=["schema"], + project_id=project_id, + dataset_id=dataset_id, + table_id=table_id, + ) + @GoogleBaseHook.fallback_to_default_project_id def poll_job_complete( self, diff --git a/airflow/providers/google/cloud/hooks/compute_ssh.py b/airflow/providers/google/cloud/hooks/compute_ssh.py index ccc538896e26b..04589316b6ab7 100644 --- a/airflow/providers/google/cloud/hooks/compute_ssh.py +++ b/airflow/providers/google/cloud/hooks/compute_ssh.py @@ -202,7 +202,7 @@ def get_conn(self) -> paramiko.SSHClient: if not self.instance_name or not self.zone or not self.project_id: raise AirflowException( f"Required parameters are missing: {missing_fields}. These parameters be passed either as " - "keyword parameter or as extra field in Airfow connection definition. Both are not set!" + "keyword parameter or as extra field in Airflow connection definition. Both are not set!" ) self.log.info( diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/airflow/providers/google/cloud/hooks/gcs.py index a4341ec87241f..63e6081d78462 100644 --- a/airflow/providers/google/cloud/hooks/gcs.py +++ b/airflow/providers/google/cloud/hooks/gcs.py @@ -265,8 +265,8 @@ def rewrite( def download( self, + bucket_name: str, object_name: str, - bucket_name: Optional[str], filename: Optional[str] = None, chunk_size: Optional[int] = None, timeout: Optional[int] = DEFAULT_TIMEOUT, @@ -280,10 +280,10 @@ def download( returns the location. For file sizes that exceed the available memory it is recommended to write to a file. - :param object_name: The object to fetch. - :type object_name: str :param bucket_name: The bucket to fetch from. :type bucket_name: str + :param object_name: The object to fetch. + :type object_name: str :param filename: If set, a local file path where the file should be written to. :type filename: str :param chunk_size: Blob chunk size. diff --git a/airflow/providers/google/cloud/operators/automl.py b/airflow/providers/google/cloud/operators/automl.py index cdf79b0ecde37..2493e67befef8 100644 --- a/airflow/providers/google/cloud/operators/automl.py +++ b/airflow/providers/google/cloud/operators/automl.py @@ -33,7 +33,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook -from airflow.utils.decorators import apply_defaults MetaData = Sequence[Tuple[str, str]] @@ -81,7 +80,6 @@ class AutoMLTrainModelOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -145,8 +143,8 @@ class AutoMLPredictOperator(BaseOperator): :type project_id: str :param location: The location of the project. :type location: str - :param params: Additional domain-specific parameters for the predictions. - :type params: Optional[Dict[str, str]] + :param operation_params: Additional domain-specific parameters for the predictions. + :type operation_params: Optional[Dict[str, str]] :param retry: A retry object used to retry requests. If `None` is specified, requests will not be retried. :type retry: Optional[google.api_core.retry.Retry] @@ -175,14 +173,13 @@ class AutoMLPredictOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, model_id: str, location: str, payload: dict, - params: Optional[Dict[str, str]] = None, + operation_params: Optional[Dict[str, str]] = None, project_id: Optional[str] = None, metadata: Optional[MetaData] = None, timeout: Optional[float] = None, @@ -194,7 +191,7 @@ def __init__( super().__init__(**kwargs) self.model_id = model_id - self.params = params # type: ignore + self.operation_params = operation_params # type: ignore self.location = location self.project_id = project_id self.metadata = metadata @@ -214,7 +211,7 @@ def execute(self, context): payload=self.payload, location=self.location, project_id=self.project_id, - params=self.params, + params=self.operation_params, retry=self.retry, timeout=self.timeout, metadata=self.metadata, @@ -283,7 +280,6 @@ class AutoMLBatchPredictOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -382,7 +378,6 @@ class AutoMLCreateDatasetOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -478,7 +473,6 @@ class AutoMLImportDataOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -583,7 +577,6 @@ class AutoMLTablesListColumnSpecsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -685,7 +678,6 @@ class AutoMLTablesUpdateDatasetOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -772,7 +764,6 @@ class AutoMLGetModelOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -858,7 +849,6 @@ class AutoMLDeleteModelOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -953,7 +943,6 @@ class AutoMLDeployModelOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1052,7 +1041,6 @@ class AutoMLTablesListTableSpecsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1142,7 +1130,6 @@ class AutoMLListDatasetOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1232,7 +1219,6 @@ class AutoMLDeleteDatasetOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 67e38f4f089fb..9d482fdf3632c 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -37,7 +37,6 @@ from airflow.operators.sql import SQLCheckOperator, SQLIntervalCheckOperator, SQLValueCheckOperator from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob from airflow.providers.google.cloud.hooks.gcs import GCSHook, _parse_gcs_url -from airflow.utils.decorators import apply_defaults BIGQUERY_JOB_DETAILS_LINK_FMT = "https://console.cloud.google.com/bigquery?j={job_id}" @@ -166,7 +165,6 @@ class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator): template_ext = ('.sql',) ui_color = BigQueryUIColors.CHECK.value - @apply_defaults def __init__( self, *, @@ -236,7 +234,6 @@ class BigQueryValueCheckOperator(_BigQueryDbHookMixin, SQLValueCheckOperator): template_ext = ('.sql',) ui_color = BigQueryUIColors.CHECK.value - @apply_defaults def __init__( self, *, @@ -321,7 +318,6 @@ class BigQueryIntervalCheckOperator(_BigQueryDbHookMixin, SQLIntervalCheckOperat ) ui_color = BigQueryUIColors.CHECK.value - @apply_defaults def __init__( self, *, @@ -429,7 +425,6 @@ class BigQueryGetDataOperator(BaseOperator): ) ui_color = BigQueryUIColors.QUERY.value - @apply_defaults def __init__( self, *, @@ -610,7 +605,6 @@ def operator_extra_links(self): return (BigQueryConsoleIndexableLink(i) for i, _ in enumerate(self.sql)) # pylint: disable=too-many-arguments, too-many-locals - @apply_defaults def __init__( self, *, @@ -889,7 +883,6 @@ class BigQueryCreateEmptyTableOperator(BaseOperator): ui_color = BigQueryUIColors.TABLE.value # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, @@ -1089,7 +1082,6 @@ class BigQueryCreateExternalTableOperator(BaseOperator): ui_color = BigQueryUIColors.TABLE.value # pylint: disable=too-many-arguments,too-many-locals - @apply_defaults def __init__( self, *, @@ -1282,7 +1274,6 @@ class BigQueryDeleteDatasetOperator(BaseOperator): ) ui_color = BigQueryUIColors.DATASET.value - @apply_defaults def __init__( self, *, @@ -1386,7 +1377,6 @@ class BigQueryCreateEmptyDatasetOperator(BaseOperator): template_fields_renderers = {"dataset_reference": "json"} ui_color = BigQueryUIColors.DATASET.value - @apply_defaults def __init__( self, *, @@ -1484,7 +1474,6 @@ class BigQueryGetDatasetOperator(BaseOperator): ) ui_color = BigQueryUIColors.DATASET.value - @apply_defaults def __init__( self, *, @@ -1555,7 +1544,6 @@ class BigQueryGetDatasetTablesOperator(BaseOperator): ) ui_color = BigQueryUIColors.DATASET.value - @apply_defaults def __init__( self, *, @@ -1635,7 +1623,6 @@ class BigQueryPatchDatasetOperator(BaseOperator): template_fields_renderers = {"dataset_resource": "json"} ui_color = BigQueryUIColors.DATASET.value - @apply_defaults def __init__( self, *, @@ -1728,7 +1715,6 @@ class BigQueryUpdateTableOperator(BaseOperator): template_fields_renderers = {"table_resource": "json"} ui_color = BigQueryUIColors.TABLE.value - @apply_defaults def __init__( self, *, @@ -1819,7 +1805,6 @@ class BigQueryUpdateDatasetOperator(BaseOperator): template_fields_renderers = {"dataset_resource": "json"} ui_color = BigQueryUIColors.DATASET.value - @apply_defaults def __init__( self, *, @@ -1901,7 +1886,6 @@ class BigQueryDeleteTableOperator(BaseOperator): ) ui_color = BigQueryUIColors.TABLE.value - @apply_defaults def __init__( self, *, @@ -1991,7 +1975,6 @@ class BigQueryUpsertTableOperator(BaseOperator): template_fields_renderers = {"table_resource": "json"} ui_color = BigQueryUIColors.TABLE.value - @apply_defaults def __init__( self, *, @@ -2039,6 +2022,117 @@ def execute(self, context) -> None: ) +class BigQueryUpdateTableSchemaOperator(BaseOperator): + """ + Update BigQuery Table Schema + Updates fields on a table schema based on contents of the supplied schema_fields_updates + parameter. The supplied schema does not need to be complete, if the field + already exists in the schema you only need to supply keys & values for the + items you want to patch, just ensure the "name" key is set. + + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:BigQueryUpdateTableSchemaOperator` + + :param schema_fields_updates: a partial schema resource. see + https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TableSchema + + **Example**: :: + + schema_fields_updates=[ + {"name": "emp_name", "description": "Some New Description"}, + {"name": "salary", "policyTags": {'names': ['some_new_policy_tag']},}, + {"name": "departments", "fields": [ + {"name": "name", "description": "Some New Description"}, + {"name": "type", "description": "Some New Description"} + ]}, + ] + + :type schema_fields_updates: List[dict] + :param include_policy_tags: (Optional) If set to True policy tags will be included in + the update request which requires special permissions even if unchanged (default False) + see https://cloud.google.com/bigquery/docs/column-level-security#roles + :type include_policy_tags: bool + :param dataset_id: A dotted + ``(.|:)`` that indicates which dataset + will be updated. (templated) + :type dataset_id: str + :param table_id: The table ID of the requested table. (templated) + :type table_id: str + :param project_id: The name of the project where we want to update the dataset. + Don't need to provide, if projectId in dataset_reference. + :type project_id: str + :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud. + :type gcp_conn_id: str + :param bigquery_conn_id: (Deprecated) The connection ID used to connect to Google Cloud. + This parameter has been deprecated. You should pass the gcp_conn_id parameter instead. + :type bigquery_conn_id: str + :param delegate_to: The account to impersonate, if any. + For this to work, the service account making the request must have domain-wide + delegation enabled. + :type delegate_to: str + :param location: The location used for the operation. + :type location: str + :param impersonation_chain: Optional service account to impersonate using short-term + credentials, or chained list of accounts required to get the access_token + of the last account in the list, which will be impersonated in the request. + If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. + If set as a sequence, the identities from the list must grant + Service Account Token Creator IAM role to the directly preceding identity, with first + account from the list granting this role to the originating account (templated). + :type impersonation_chain: Union[str, Sequence[str]] + """ + + template_fields = ( + 'schema_fields_updates', + 'dataset_id', + 'table_id', + 'project_id', + 'impersonation_chain', + ) + template_fields_renderers = {"schema_fields_updates": "json"} + ui_color = BigQueryUIColors.TABLE.value + + def __init__( + self, + *, + schema_fields_updates: List[Dict[str, Any]], + include_policy_tags: Optional[bool] = False, + dataset_id: Optional[str] = None, + table_id: Optional[str] = None, + project_id: Optional[str] = None, + gcp_conn_id: str = 'google_cloud_default', + delegate_to: Optional[str] = None, + impersonation_chain: Optional[Union[str, Sequence[str]]] = None, + **kwargs, + ) -> None: + self.schema_fields_updates = schema_fields_updates + self.include_policy_tags = include_policy_tags + self.table_id = table_id + self.dataset_id = dataset_id + self.project_id = project_id + self.gcp_conn_id = gcp_conn_id + self.delegate_to = delegate_to + self.impersonation_chain = impersonation_chain + super().__init__(**kwargs) + + def execute(self, context): + bq_hook = BigQueryHook( + gcp_conn_id=self.gcp_conn_id, + delegate_to=self.delegate_to, + impersonation_chain=self.impersonation_chain, + ) + + return bq_hook.update_table_schema( + schema_fields_updates=self.schema_fields_updates, + include_policy_tags=self.include_policy_tags, + dataset_id=self.dataset_id, + table_id=self.table_id, + project_id=self.project_id, + ) + + # pylint: disable=too-many-arguments class BigQueryInsertJobOperator(BaseOperator): """ @@ -2109,7 +2203,6 @@ class BigQueryInsertJobOperator(BaseOperator): template_fields_renderers = {"configuration": "json"} ui_color = BigQueryUIColors.QUERY.value - @apply_defaults def __init__( self, configuration: Dict[str, Any], diff --git a/airflow/providers/google/cloud/operators/bigquery_dts.py b/airflow/providers/google/cloud/operators/bigquery_dts.py index 656fc775bb003..7b884c0068e23 100644 --- a/airflow/providers/google/cloud/operators/bigquery_dts.py +++ b/airflow/providers/google/cloud/operators/bigquery_dts.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook, get_object_id -from airflow.utils.decorators import apply_defaults class BigQueryCreateDataTransferOperator(BaseOperator): @@ -73,7 +72,6 @@ class BigQueryCreateDataTransferOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -158,7 +156,6 @@ class BigQueryDeleteDataTransferConfigOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -249,7 +246,6 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/bigtable.py b/airflow/providers/google/cloud/operators/bigtable.py index 6df02207dbfc6..7c97187b430b8 100644 --- a/airflow/providers/google/cloud/operators/bigtable.py +++ b/airflow/providers/google/cloud/operators/bigtable.py @@ -26,7 +26,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.bigtable import BigtableHook -from airflow.utils.decorators import apply_defaults class BigtableValidationMixin: @@ -110,7 +109,6 @@ class BigtableCreateInstanceOperator(BaseOperator, BigtableValidationMixin): 'impersonation_chain', ] - @apply_defaults def __init__( self, *, # pylint: disable=too-many-arguments @@ -230,7 +228,6 @@ class BigtableUpdateInstanceOperator(BaseOperator, BigtableValidationMixin): 'impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -314,7 +311,6 @@ class BigtableDeleteInstanceOperator(BaseOperator, BigtableValidationMixin): 'impersonation_chain', ] # type: Iterable[str] - @apply_defaults def __init__( self, *, @@ -396,7 +392,6 @@ class BigtableCreateTableOperator(BaseOperator, BigtableValidationMixin): 'impersonation_chain', ] # type: Iterable[str] - @apply_defaults def __init__( self, *, @@ -508,7 +503,6 @@ class BigtableDeleteTableOperator(BaseOperator, BigtableValidationMixin): 'impersonation_chain', ] # type: Iterable[str] - @apply_defaults def __init__( self, *, @@ -594,7 +588,6 @@ class BigtableUpdateClusterOperator(BaseOperator, BigtableValidationMixin): 'impersonation_chain', ] # type: Iterable[str] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/cloud_build.py b/airflow/providers/google/cloud/operators/cloud_build.py index b4c0cf72bf7f9..8e30d983cf786 100644 --- a/airflow/providers/google/cloud/operators/cloud_build.py +++ b/airflow/providers/google/cloud/operators/cloud_build.py @@ -30,7 +30,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.cloud_build import CloudBuildHook -from airflow.utils.decorators import apply_defaults REGEX_REPO_PATH = re.compile(r"^/p/(?P[^/]+)/r/(?P[^/]+)") @@ -200,7 +199,6 @@ class CloudBuildCreateBuildOperator(BaseOperator): ) template_ext = ['.yml', '.yaml', '.json'] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/cloud_memorystore.py b/airflow/providers/google/cloud/operators/cloud_memorystore.py index 64a6251992048..9214a9211f282 100644 --- a/airflow/providers/google/cloud/operators/cloud_memorystore.py +++ b/airflow/providers/google/cloud/operators/cloud_memorystore.py @@ -28,7 +28,6 @@ CloudMemorystoreHook, CloudMemorystoreMemcachedHook, ) -from airflow.utils.decorators import apply_defaults class CloudMemorystoreCreateInstanceOperator(BaseOperator): @@ -94,7 +93,6 @@ class CloudMemorystoreCreateInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -183,7 +181,6 @@ class CloudMemorystoreDeleteInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -275,7 +272,6 @@ class CloudMemorystoreExportInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -368,7 +364,6 @@ class CloudMemorystoreFailoverInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -455,7 +450,6 @@ class CloudMemorystoreGetInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -549,7 +543,6 @@ class CloudMemorystoreImportOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -640,7 +633,6 @@ class CloudMemorystoreListInstancesOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -746,7 +738,6 @@ class CloudMemorystoreUpdateInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -840,7 +831,6 @@ class CloudMemorystoreScaleInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -953,7 +943,6 @@ class CloudMemorystoreCreateInstanceAndImportOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1062,7 +1051,6 @@ class CloudMemorystoreExportAndDeleteInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1157,7 +1145,6 @@ class CloudMemorystoreMemcachedApplyParametersOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1254,7 +1241,6 @@ class CloudMemorystoreMemcachedCreateInstanceOperator(BaseOperator): "gcp_conn_id", ) - @apply_defaults def __init__( self, location: str, @@ -1321,7 +1307,6 @@ class CloudMemorystoreMemcachedDeleteInstanceOperator(BaseOperator): template_fields = ("location", "instance", "project_id", "retry", "timeout", "metadata", "gcp_conn_id") - @apply_defaults def __init__( self, location: str, @@ -1401,7 +1386,6 @@ class CloudMemorystoreMemcachedGetInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1486,7 +1470,6 @@ class CloudMemorystoreMemcachedListInstancesOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1586,7 +1569,6 @@ class CloudMemorystoreMemcachedUpdateInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1679,7 +1661,6 @@ class CloudMemorystoreMemcachedUpdateParametersOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index 6249041b67bd6..91086b3c6a7aa 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -27,7 +27,6 @@ from airflow.providers.google.cloud.utils.field_validator import GcpBodyFieldValidator from airflow.providers.mysql.hooks.mysql import MySqlHook from airflow.providers.postgres.hooks.postgres import PostgresHook -from airflow.utils.decorators import apply_defaults SETTINGS = 'settings' SETTINGS_VERSION = 'settingsVersion' @@ -211,7 +210,6 @@ class CloudSQLBaseOperator(BaseOperator): :type impersonation_chain: Union[str, Sequence[str]] """ - @apply_defaults def __init__( self, *, @@ -309,7 +307,6 @@ class CloudSQLCreateInstanceOperator(CloudSQLBaseOperator): ) # [END gcp_sql_create_template_fields] - @apply_defaults def __init__( self, *, @@ -411,7 +408,6 @@ class CloudSQLInstancePatchOperator(CloudSQLBaseOperator): ) # [END gcp_sql_patch_template_fields] - @apply_defaults def __init__( self, *, @@ -491,26 +487,6 @@ class CloudSQLDeleteInstanceOperator(CloudSQLBaseOperator): ) # [END gcp_sql_delete_template_fields] - @apply_defaults - def __init__( - self, - *, - instance: str, - project_id: Optional[str] = None, - gcp_conn_id: str = 'google_cloud_default', - api_version: str = 'v1beta4', - impersonation_chain: Optional[Union[str, Sequence[str]]] = None, - **kwargs, - ) -> None: - super().__init__( - project_id=project_id, - instance=instance, - gcp_conn_id=gcp_conn_id, - api_version=api_version, - impersonation_chain=impersonation_chain, - **kwargs, - ) - def execute(self, context) -> Optional[bool]: hook = CloudSQLHook( gcp_conn_id=self.gcp_conn_id, @@ -568,7 +544,6 @@ class CloudSQLCreateInstanceDatabaseOperator(CloudSQLBaseOperator): ) # [END gcp_sql_db_create_template_fields] - @apply_defaults def __init__( self, *, @@ -677,7 +652,6 @@ class CloudSQLPatchInstanceDatabaseOperator(CloudSQLBaseOperator): ) # [END gcp_sql_db_patch_template_fields] - @apply_defaults def __init__( self, *, @@ -778,7 +752,6 @@ class CloudSQLDeleteInstanceDatabaseOperator(CloudSQLBaseOperator): ) # [END gcp_sql_db_delete_template_fields] - @apply_defaults def __init__( self, *, @@ -871,7 +844,6 @@ class CloudSQLExportInstanceOperator(CloudSQLBaseOperator): ) # [END gcp_sql_export_template_fields] - @apply_defaults def __init__( self, *, @@ -976,7 +948,6 @@ class CloudSQLImportInstanceOperator(CloudSQLBaseOperator): ) # [END gcp_sql_import_template_fields] - @apply_defaults def __init__( self, *, @@ -1056,7 +1027,6 @@ class CloudSQLExecuteQueryOperator(BaseOperator): template_ext = ('.sql',) # [END gcp_sql_query_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py b/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py index c021c9f9ac15c..6c803ef35a24f 100644 --- a/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +++ b/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py @@ -53,7 +53,6 @@ CloudDataTransferServiceHook, GcpTransferJobsStatus, ) -from airflow.utils.decorators import apply_defaults class TransferJobPreprocessor: @@ -219,7 +218,6 @@ class CloudDataTransferServiceCreateJobOperator(BaseOperator): ) # [END gcp_transfer_job_create_template_fields] - @apply_defaults def __init__( self, *, @@ -299,7 +297,6 @@ class CloudDataTransferServiceUpdateJobOperator(BaseOperator): ) # [END gcp_transfer_job_update_template_fields] - @apply_defaults def __init__( self, *, @@ -377,7 +374,6 @@ class CloudDataTransferServiceDeleteJobOperator(BaseOperator): ) # [END gcp_transfer_job_delete_template_fields] - @apply_defaults def __init__( self, *, @@ -445,7 +441,6 @@ class CloudDataTransferServiceGetOperationOperator(BaseOperator): ) # [END gcp_transfer_operation_get_template_fields] - @apply_defaults def __init__( self, *, @@ -585,7 +580,6 @@ class CloudDataTransferServicePauseOperationOperator(BaseOperator): ) # [END gcp_transfer_operation_pause_template_fields] - @apply_defaults def __init__( self, *, @@ -649,7 +643,6 @@ class CloudDataTransferServiceResumeOperationOperator(BaseOperator): ) # [END gcp_transfer_operation_resume_template_fields] - @apply_defaults def __init__( self, *, @@ -714,7 +707,6 @@ class CloudDataTransferServiceCancelOperationOperator(BaseOperator): ) # [END gcp_transfer_operation_cancel_template_fields] - @apply_defaults def __init__( self, *, @@ -828,7 +820,6 @@ class CloudDataTransferServiceS3ToGCSOperator(BaseOperator): ) ui_color = '#e09411' - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -998,7 +989,6 @@ class CloudDataTransferServiceGCSToGCSOperator(BaseOperator): ) ui_color = '#e09411' - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/google/cloud/operators/compute.py b/airflow/providers/google/cloud/operators/compute.py index 83f150d0f40d8..694e5d10fde04 100644 --- a/airflow/providers/google/cloud/operators/compute.py +++ b/airflow/providers/google/cloud/operators/compute.py @@ -28,13 +28,11 @@ from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.utils.field_sanitizer import GcpBodyFieldSanitizer from airflow.providers.google.cloud.utils.field_validator import GcpBodyFieldValidator -from airflow.utils.decorators import apply_defaults class ComputeEngineBaseOperator(BaseOperator): """Abstract base operator for Google Compute Engine operators to inherit from.""" - @apply_defaults def __init__( self, *, @@ -111,28 +109,6 @@ class ComputeEngineStartInstanceOperator(ComputeEngineBaseOperator): ) # [END gce_instance_start_template_fields] - @apply_defaults - def __init__( - self, - *, - zone: str, - resource_id: str, - project_id: Optional[str] = None, - gcp_conn_id: str = 'google_cloud_default', - api_version: str = 'v1', - impersonation_chain: Optional[Union[str, Sequence[str]]] = None, - **kwargs, - ) -> None: - super().__init__( - project_id=project_id, - zone=zone, - resource_id=resource_id, - gcp_conn_id=gcp_conn_id, - api_version=api_version, - impersonation_chain=impersonation_chain, - **kwargs, - ) - def execute(self, context) -> None: hook = ComputeEngineHook( gcp_conn_id=self.gcp_conn_id, @@ -186,28 +162,6 @@ class ComputeEngineStopInstanceOperator(ComputeEngineBaseOperator): ) # [END gce_instance_stop_template_fields] - @apply_defaults - def __init__( - self, - *, - zone: str, - resource_id: str, - project_id: Optional[str] = None, - gcp_conn_id: str = 'google_cloud_default', - api_version: str = 'v1', - impersonation_chain: Optional[Union[str, Sequence[str]]] = None, - **kwargs, - ) -> None: - super().__init__( - project_id=project_id, - zone=zone, - resource_id=resource_id, - gcp_conn_id=gcp_conn_id, - api_version=api_version, - impersonation_chain=impersonation_chain, - **kwargs, - ) - def execute(self, context) -> None: hook = ComputeEngineHook( gcp_conn_id=self.gcp_conn_id, @@ -274,7 +228,6 @@ class ComputeEngineSetMachineTypeOperator(ComputeEngineBaseOperator): ) # [END gce_instance_set_machine_type_template_fields] - @apply_defaults def __init__( self, *, @@ -437,7 +390,6 @@ class ComputeEngineCopyInstanceTemplateOperator(ComputeEngineBaseOperator): ) # [END gce_instance_template_copy_operator_template_fields] - @apply_defaults def __init__( self, *, @@ -574,7 +526,6 @@ class ComputeEngineInstanceGroupUpdateManagerTemplateOperator(ComputeEngineBaseO ) # [END gce_igm_update_template_operator_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/datacatalog.py b/airflow/providers/google/cloud/operators/datacatalog.py index fcb8ccbad2f3b..c33efa82eb9e7 100644 --- a/airflow/providers/google/cloud/operators/datacatalog.py +++ b/airflow/providers/google/cloud/operators/datacatalog.py @@ -32,7 +32,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.datacatalog import CloudDataCatalogHook -from airflow.utils.decorators import apply_defaults class CloudDataCatalogCreateEntryOperator(BaseOperator): @@ -95,7 +94,6 @@ class CloudDataCatalogCreateEntryOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -213,7 +211,6 @@ class CloudDataCatalogCreateEntryGroupOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -331,7 +328,6 @@ class CloudDataCatalogCreateTagOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -462,7 +458,6 @@ class CloudDataCatalogCreateTagTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -580,7 +575,6 @@ class CloudDataCatalogCreateTagTemplateFieldOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -691,7 +685,6 @@ class CloudDataCatalogDeleteEntryOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -785,7 +778,6 @@ class CloudDataCatalogDeleteEntryGroupOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -880,7 +872,6 @@ class CloudDataCatalogDeleteTagOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -980,7 +971,6 @@ class CloudDataCatalogDeleteTagTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1078,7 +1068,6 @@ class CloudDataCatalogDeleteTagTemplateFieldOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1176,7 +1165,6 @@ class CloudDataCatalogGetEntryOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1272,7 +1260,6 @@ class CloudDataCatalogGetEntryGroupOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1362,7 +1349,6 @@ class CloudDataCatalogGetTagTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1458,7 +1444,6 @@ class CloudDataCatalogListTagsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1553,7 +1538,6 @@ class CloudDataCatalogLookupEntryOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1647,7 +1631,6 @@ class CloudDataCatalogRenameTagTemplateFieldOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1769,7 +1752,6 @@ class CloudDataCatalogSearchCatalogOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1875,7 +1857,6 @@ class CloudDataCatalogUpdateEntryOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -1987,7 +1968,6 @@ class CloudDataCatalogUpdateTagOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -2103,7 +2083,6 @@ class CloudDataCatalogUpdateTagTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2221,7 +2200,6 @@ class CloudDataCatalogUpdateTagTemplateFieldOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/google/cloud/operators/dataflow.py b/airflow/providers/google/cloud/operators/dataflow.py index 93816e7f2b5ef..cb31c11879ecb 100644 --- a/airflow/providers/google/cloud/operators/dataflow.py +++ b/airflow/providers/google/cloud/operators/dataflow.py @@ -31,7 +31,6 @@ process_line_and_extract_dataflow_job_id_callback, ) from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults from airflow.version import version @@ -349,7 +348,6 @@ class DataflowCreateJavaJobOperator(BaseOperator): ui_color = "#0273d4" # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, @@ -635,7 +633,6 @@ class DataflowTemplatedJobStartOperator(BaseOperator): ] ui_color = "#0273d4" - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -774,7 +771,6 @@ class DataflowStartFlexTemplateOperator(BaseOperator): template_fields = ["body", "location", "project_id", "gcp_conn_id"] - @apply_defaults def __init__( self, body: Dict, @@ -878,7 +874,6 @@ class DataflowStartSqlJobOperator(BaseOperator): "gcp_conn_id", ] - @apply_defaults def __init__( self, job_name: str, @@ -1051,7 +1046,6 @@ class DataflowCreatePythonJobOperator(BaseOperator): template_fields = ["options", "dataflow_default_options", "job_name", "py_file"] - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/google/cloud/operators/datafusion.py b/airflow/providers/google/cloud/operators/datafusion.py index d302a08d2001c..c8283a60ca8d2 100644 --- a/airflow/providers/google/cloud/operators/datafusion.py +++ b/airflow/providers/google/cloud/operators/datafusion.py @@ -24,7 +24,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.datafusion import DataFusionHook -from airflow.utils.decorators import apply_defaults class CloudDataFusionRestartInstanceOperator(BaseOperator): @@ -66,7 +65,6 @@ class CloudDataFusionRestartInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -143,7 +141,6 @@ class CloudDataFusionDeleteInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -224,7 +221,6 @@ class CloudDataFusionCreateInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -332,7 +328,6 @@ class CloudDataFusionUpdateInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -415,7 +410,6 @@ class CloudDataFusionGetInstanceOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -499,7 +493,6 @@ class CloudDataFusionCreatePipelineOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -596,7 +589,6 @@ class CloudDataFusionDeletePipelineOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -694,7 +686,6 @@ class CloudDataFusionListPipelinesOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -798,7 +789,6 @@ class CloudDataFusionStartPipelineOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -906,7 +896,6 @@ class CloudDataFusionStopPipelineOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/dataprep.py b/airflow/providers/google/cloud/operators/dataprep.py index 5b806d9c55998..ee2fcfa378b27 100644 --- a/airflow/providers/google/cloud/operators/dataprep.py +++ b/airflow/providers/google/cloud/operators/dataprep.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.dataprep import GoogleDataprepHook -from airflow.utils.decorators import apply_defaults class DataprepGetJobsForJobGroupOperator(BaseOperator): @@ -37,7 +36,6 @@ class DataprepGetJobsForJobGroupOperator(BaseOperator): template_fields = ("job_id",) - @apply_defaults def __init__(self, *, dataprep_conn_id: str = "dataprep_default", job_id: int, **kwargs) -> None: super().__init__(**kwargs) self.dataprep_conn_id = (dataprep_conn_id,) @@ -72,7 +70,6 @@ class DataprepGetJobGroupOperator(BaseOperator): template_fields = ("job_group_id", "embed") - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index c8ee5e35a69f1..fea5acc883b7a 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -35,11 +35,55 @@ from google.protobuf.field_mask_pb2 import FieldMask from airflow.exceptions import AirflowException -from airflow.models import BaseOperator +from airflow.models import BaseOperator, BaseOperatorLink +from airflow.models.taskinstance import TaskInstance from airflow.providers.google.cloud.hooks.dataproc import DataprocHook, DataProcJobBuilder from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults + +DATAPROC_BASE_LINK = "https://console.cloud.google.com/dataproc" +DATAPROC_JOB_LOG_LINK = DATAPROC_BASE_LINK + "/jobs/{job_id}?region={region}&project={project_id}" +DATAPROC_CLUSTER_LINK = ( + DATAPROC_BASE_LINK + "/clusters/{cluster_name}/monitoring?region={region}&project={project_id}" +) + + +class DataprocJobLink(BaseOperatorLink): + """Helper class for constructing Dataproc Job link""" + + name = "Dataproc Job" + + def get_link(self, operator, dttm): + ti = TaskInstance(task=operator, execution_date=dttm) + job_conf = ti.xcom_pull(task_ids=operator.task_id, key="job_conf") + return ( + DATAPROC_JOB_LOG_LINK.format( + job_id=job_conf["job_id"], + region=job_conf["region"], + project_id=job_conf["project_id"], + ) + if job_conf + else "" + ) + + +class DataprocClusterLink(BaseOperatorLink): + """Helper class for constructing Dataproc Cluster link""" + + name = "Dataproc Cluster" + + def get_link(self, operator, dttm): + ti = TaskInstance(task=operator, execution_date=dttm) + cluster_conf = ti.xcom_pull(task_ids=operator.task_id, key="cluster_conf") + return ( + DATAPROC_CLUSTER_LINK.format( + cluster_name=cluster_conf["cluster_name"], + region=cluster_conf["region"], + project_id=cluster_conf["project_id"], + ) + if cluster_conf + else "" + ) # pylint: disable=too-many-instance-attributes @@ -478,7 +522,8 @@ class DataprocCreateClusterOperator(BaseOperator): ) template_fields_renderers = {'cluster_config': 'json'} - @apply_defaults + operator_extra_links = (DataprocClusterLink(),) + def __init__( # pylint: disable=too-many-arguments self, *, @@ -620,6 +665,16 @@ def _wait_for_cluster_in_creating_state(self, hook: DataprocHook) -> Cluster: def execute(self, context) -> dict: self.log.info('Creating cluster: %s', self.cluster_name) hook = DataprocHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain) + # Save data required to display extra link no matter what the cluster status will be + self.xcom_push( + context, + key="cluster_conf", + value={ + "cluster_name": self.cluster_name, + "region": self.region, + "project_id": self.project_id, + }, + ) try: # First try to create a new cluster cluster = self._create_cluster(hook) @@ -694,7 +749,8 @@ class DataprocScaleClusterOperator(BaseOperator): template_fields = ['cluster_name', 'project_id', 'region', 'impersonation_chain'] - @apply_defaults + operator_extra_links = (DataprocClusterLink(),) + def __init__( self, *, @@ -773,6 +829,16 @@ def execute(self, context) -> None: update_mask = ["config.worker_config.num_instances", "config.secondary_worker_config.num_instances"] hook = DataprocHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain) + # Save data required to display extra link no matter what the cluster status will be + self.xcom_push( + context, + key="cluster_conf", + value={ + "cluster_name": self.cluster_name, + "region": self.region, + "project_id": self.project_id, + }, + ) operation = hook.update_cluster( project_id=self.project_id, location=self.region, @@ -825,7 +891,6 @@ class DataprocDeleteClusterOperator(BaseOperator): template_fields = ('project_id', 'region', 'cluster_name', 'impersonation_chain') - @apply_defaults def __init__( self, *, @@ -931,7 +996,8 @@ class DataprocJobBaseOperator(BaseOperator): job_type = "" - @apply_defaults + operator_extra_links = (DataprocJobLink(),) + def __init__( self, *, @@ -1005,6 +1071,12 @@ def execute(self, context): ) job_id = job_object.reference.job_id self.log.info('Job %s submitted successfully.', job_id) + # Save data required for extra links no matter what the job status will be + self.xcom_push( + context, + key='job_conf', + value={'job_id': job_id, 'region': self.region, 'project_id': self.project_id}, + ) if not self.asynchronous: self.log.info('Waiting for job %s to complete', job_id) @@ -1082,7 +1154,8 @@ class DataprocSubmitPigJobOperator(DataprocJobBaseOperator): ui_color = '#0273d4' job_type = 'pig_job' - @apply_defaults + operator_extra_links = (DataprocJobLink(),) + def __init__( self, *, @@ -1157,7 +1230,6 @@ class DataprocSubmitHiveJobOperator(DataprocJobBaseOperator): ui_color = '#0273d4' job_type = 'hive_job' - @apply_defaults def __init__( self, *, @@ -1232,7 +1304,6 @@ class DataprocSubmitSparkSqlJobOperator(DataprocJobBaseOperator): ui_color = '#0273d4' job_type = 'spark_sql_job' - @apply_defaults def __init__( self, *, @@ -1312,7 +1383,6 @@ class DataprocSubmitSparkJobOperator(DataprocJobBaseOperator): ui_color = '#0273d4' job_type = 'spark_job' - @apply_defaults def __init__( self, *, @@ -1392,7 +1462,6 @@ class DataprocSubmitHadoopJobOperator(DataprocJobBaseOperator): ui_color = '#0273d4' job_type = 'hadoop_job' - @apply_defaults def __init__( self, *, @@ -1497,7 +1566,6 @@ def _upload_file_temp(self, bucket, local_file): ) return f"gs://{bucket}/{temp_filename}" - @apply_defaults def __init__( self, *, @@ -1679,7 +1747,6 @@ class DataprocInstantiateWorkflowTemplateOperator(BaseOperator): template_fields = ['template_id', 'impersonation_chain', 'request_id', 'parameters'] template_fields_renderers = {"parameters": "json"} - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -1779,7 +1846,6 @@ class DataprocInstantiateInlineWorkflowTemplateOperator(BaseOperator): template_fields = ['template', 'impersonation_chain'] template_fields_renderers = {"template": "json"} - @apply_defaults def __init__( self, *, @@ -1871,7 +1937,8 @@ class DataprocSubmitJobOperator(BaseOperator): template_fields = ('project_id', 'location', 'job', 'impersonation_chain', 'request_id') template_fields_renderers = {"job": "json"} - @apply_defaults + operator_extra_links = (DataprocJobLink(),) + def __init__( self, *, @@ -1919,6 +1986,16 @@ def execute(self, context: Dict): ) job_id = job_object.reference.job_id self.log.info('Job %s submitted successfully.', job_id) + # Save data required by extra links no matter what the job status will be + self.xcom_push( + context, + key="job_conf", + value={ + "job_id": job_id, + "region": self.location, + "project_id": self.project_id, + }, + ) if not self.asynchronous: self.log.info('Waiting for job %s to complete', job_id) @@ -1988,8 +2065,8 @@ class DataprocUpdateClusterOperator(BaseOperator): """ template_fields = ('impersonation_chain', 'cluster_name') + operator_extra_links = (DataprocClusterLink(),) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -2023,6 +2100,16 @@ def __init__( # pylint: disable=too-many-arguments def execute(self, context: Dict): hook = DataprocHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain) + # Save data required by extra links no matter what the cluster status will be + self.xcom_push( + context, + key="cluster_conf", + value={ + "cluster_name": self.cluster_name, + "region": self.location, + "project_id": self.project_id, + }, + ) self.log.info("Updating %s cluster.", self.cluster_name) operation = hook.update_cluster( project_id=self.project_id, diff --git a/airflow/providers/google/cloud/operators/datastore.py b/airflow/providers/google/cloud/operators/datastore.py index 234a289643377..e300d7b35b85b 100644 --- a/airflow/providers/google/cloud/operators/datastore.py +++ b/airflow/providers/google/cloud/operators/datastore.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.datastore import DatastoreHook from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults class CloudDatastoreExportEntitiesOperator(BaseOperator): @@ -79,7 +78,6 @@ class CloudDatastoreExportEntitiesOperator(BaseOperator): 'impersonation_chain', ] - @apply_defaults def __init__( self, # pylint: disable=too-many-arguments *, @@ -192,7 +190,6 @@ class CloudDatastoreImportEntitiesOperator(BaseOperator): 'impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -284,7 +281,6 @@ class CloudDatastoreAllocateIdsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -352,7 +348,6 @@ class CloudDatastoreBeginTransactionOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -420,7 +415,6 @@ class CloudDatastoreCommitOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -488,7 +482,6 @@ class CloudDatastoreRollbackOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -555,7 +548,6 @@ class CloudDatastoreRunQueryOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -617,7 +609,6 @@ class CloudDatastoreGetOperationOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -674,7 +665,6 @@ class CloudDatastoreDeleteOperationOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/dlp.py b/airflow/providers/google/cloud/operators/dlp.py index 099feb2f1aed0..42564d4d08b3a 100644 --- a/airflow/providers/google/cloud/operators/dlp.py +++ b/airflow/providers/google/cloud/operators/dlp.py @@ -44,7 +44,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.dlp import CloudDLPHook -from airflow.utils.decorators import apply_defaults class CloudDLPCancelDLPJobOperator(BaseOperator): @@ -90,7 +89,6 @@ class CloudDLPCancelDLPJobOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -179,7 +177,6 @@ class CloudDLPCreateDeidentifyTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -287,7 +284,6 @@ class CloudDLPCreateDLPJobOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -395,7 +391,6 @@ class CloudDLPCreateInspectTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -497,7 +492,6 @@ class CloudDLPCreateJobTriggerOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -600,7 +594,6 @@ class CloudDLPCreateStoredInfoTypeOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -719,7 +712,6 @@ class CloudDLPDeidentifyContentOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -815,7 +807,6 @@ class CloudDLPDeleteDeidentifyTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -901,7 +892,6 @@ class CloudDLPDeleteDLPJobOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -987,7 +977,6 @@ class CloudDLPDeleteInspectTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1072,7 +1061,6 @@ class CloudDLPDeleteJobTriggerOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1158,7 +1146,6 @@ class CloudDLPDeleteStoredInfoTypeOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1249,7 +1236,6 @@ class CloudDLPGetDeidentifyTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1334,7 +1320,6 @@ class CloudDLPGetDLPJobOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1420,7 +1405,6 @@ class CloudDLPGetInspectTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1505,7 +1489,6 @@ class CloudDLPGetDLPJobTriggerOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1591,7 +1574,6 @@ class CloudDLPGetStoredInfoTypeOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1685,7 +1667,6 @@ class CloudDLPInspectContentOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1780,7 +1761,6 @@ class CloudDLPListDeidentifyTemplatesOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1875,7 +1855,6 @@ class CloudDLPListDLPJobsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1965,7 +1944,6 @@ class CloudDLPListInfoTypesOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2054,7 +2032,6 @@ class CloudDLPListInspectTemplatesOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2147,7 +2124,6 @@ class CloudDLPListJobTriggersOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2242,7 +2218,6 @@ class CloudDLPListStoredInfoTypesOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2344,7 +2319,6 @@ class CloudDLPRedactImageOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2451,7 +2425,6 @@ class CloudDLPReidentifyContentOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2555,7 +2528,6 @@ class CloudDLPUpdateDeidentifyTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2656,7 +2628,6 @@ class CloudDLPUpdateInspectTemplateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2753,7 +2724,6 @@ class CloudDLPUpdateJobTriggerOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -2852,7 +2822,6 @@ class CloudDLPUpdateStoredInfoTypeOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/functions.py b/airflow/providers/google/cloud/operators/functions.py index 451748433271d..44034315131d8 100644 --- a/airflow/providers/google/cloud/operators/functions.py +++ b/airflow/providers/google/cloud/operators/functions.py @@ -29,7 +29,6 @@ GcpBodyFieldValidator, GcpFieldValidationException, ) -from airflow.utils.decorators import apply_defaults from airflow.version import version @@ -150,7 +149,6 @@ class CloudFunctionDeployFunctionOperator(BaseOperator): ) # [END gcf_function_deploy_template_fields] - @apply_defaults def __init__( self, *, @@ -360,7 +358,6 @@ class CloudFunctionDeleteFunctionOperator(BaseOperator): ) # [END gcf_function_delete_template_fields] - @apply_defaults def __init__( self, *, @@ -442,7 +439,6 @@ class CloudFunctionInvokeFunctionOperator(BaseOperator): 'impersonation_chain', ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/gcs.py b/airflow/providers/google/cloud/operators/gcs.py index 8c8f241690fba..5fa1eb2fd11d5 100644 --- a/airflow/providers/google/cloud/operators/gcs.py +++ b/airflow/providers/google/cloud/operators/gcs.py @@ -31,7 +31,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults class GCSCreateBucketOperator(BaseOperator): @@ -116,7 +115,6 @@ class GCSCreateBucketOperator(BaseOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, @@ -229,7 +227,6 @@ class GCSListObjectsOperator(BaseOperator): ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, @@ -318,7 +315,6 @@ class GCSDeleteObjectsOperator(BaseOperator): 'impersonation_chain', ) - @apply_defaults def __init__( self, *, @@ -416,7 +412,6 @@ class GCSBucketCreateAclEntryOperator(BaseOperator): ) # [END gcs_bucket_create_acl_template_fields] - @apply_defaults def __init__( self, *, @@ -511,7 +506,6 @@ class GCSObjectCreateAclEntryOperator(BaseOperator): ) # [END gcs_object_create_acl_template_fields] - @apply_defaults def __init__( self, *, @@ -602,7 +596,6 @@ class GCSFileTransformOperator(BaseOperator): 'impersonation_chain', ) - @apply_defaults def __init__( self, *, @@ -759,7 +752,6 @@ def interpolate_prefix(prefix: str, dt: datetime.datetime) -> Optional[datetime. """ return dt.strftime(prefix) if prefix else None - @apply_defaults def __init__( self, *, @@ -942,7 +934,6 @@ class GCSDeleteBucketOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1029,7 +1020,6 @@ class GCSSynchronizeBucketsOperator(BaseOperator): 'impersonation_chain', ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/kubernetes_engine.py b/airflow/providers/google/cloud/operators/kubernetes_engine.py index 407c76a19a600..c763fb0d621c0 100644 --- a/airflow/providers/google/cloud/operators/kubernetes_engine.py +++ b/airflow/providers/google/cloud/operators/kubernetes_engine.py @@ -29,7 +29,6 @@ from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator from airflow.providers.google.cloud.hooks.kubernetes_engine import GKEHook from airflow.providers.google.common.hooks.base_google import GoogleBaseHook -from airflow.utils.decorators import apply_defaults from airflow.utils.process_utils import execute_in_subprocess, patch_environ @@ -87,7 +86,6 @@ class GKEDeleteClusterOperator(BaseOperator): 'impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -190,7 +188,6 @@ class GKECreateClusterOperator(BaseOperator): 'impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -272,7 +269,6 @@ class GKEStartPodOperator(KubernetesPodOperator): template_fields = {'project_id', 'location', 'cluster_name'} | set(KubernetesPodOperator.template_fields) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/life_sciences.py b/airflow/providers/google/cloud/operators/life_sciences.py index 1f06927d9a6dd..763365478b536 100644 --- a/airflow/providers/google/cloud/operators/life_sciences.py +++ b/airflow/providers/google/cloud/operators/life_sciences.py @@ -22,7 +22,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.life_sciences import LifeSciencesHook -from airflow.utils.decorators import apply_defaults class LifeSciencesRunPipelineOperator(BaseOperator): @@ -62,7 +61,6 @@ class LifeSciencesRunPipelineOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/mlengine.py b/airflow/providers/google/cloud/operators/mlengine.py index b9f918466d720..b8de5253805c9 100644 --- a/airflow/providers/google/cloud/operators/mlengine.py +++ b/airflow/providers/google/cloud/operators/mlengine.py @@ -25,7 +25,6 @@ from airflow.models import BaseOperator, BaseOperatorLink from airflow.models.taskinstance import TaskInstance from airflow.providers.google.cloud.hooks.mlengine import MLEngineHook -from airflow.utils.decorators import apply_defaults log = logging.getLogger(__name__) @@ -177,7 +176,6 @@ class MLEngineStartBatchPredictionJobOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, # pylint: disable=too-many-arguments *, @@ -341,7 +339,6 @@ class MLEngineManageModelOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -422,7 +419,6 @@ class MLEngineCreateModelOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -488,7 +484,6 @@ class MLEngineGetModelOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -558,7 +553,6 @@ class MLEngineDeleteModelOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -662,7 +656,6 @@ class MLEngineManageVersionOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -766,7 +759,6 @@ class MLEngineCreateVersionOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -850,7 +842,6 @@ class MLEngineSetDefaultVersionOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -931,7 +922,6 @@ class MLEngineListVersionsOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -1011,7 +1001,6 @@ class MLEngineDeleteVersionOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, @@ -1167,7 +1156,6 @@ class MLEngineStartTrainingJobOperator(BaseOperator): operator_extra_links = (AIPlatformConsoleLink(),) - @apply_defaults def __init__( self, # pylint: disable=too-many-arguments *, @@ -1352,7 +1340,6 @@ class MLEngineTrainingCancelJobOperator(BaseOperator): '_impersonation_chain', ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/natural_language.py b/airflow/providers/google/cloud/operators/natural_language.py index fa9e89e3d4362..6fc16772f71db 100644 --- a/airflow/providers/google/cloud/operators/natural_language.py +++ b/airflow/providers/google/cloud/operators/natural_language.py @@ -25,7 +25,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.natural_language import CloudNaturalLanguageHook -from airflow.utils.decorators import apply_defaults MetaData = Sequence[Tuple[str, str]] @@ -72,7 +71,6 @@ class CloudNaturalLanguageAnalyzeEntitiesOperator(BaseOperator): ) # [END natural_language_analyze_entities_template_fields] - @apply_defaults def __init__( self, *, @@ -153,7 +151,6 @@ class CloudNaturalLanguageAnalyzeEntitySentimentOperator(BaseOperator): ) # [END natural_language_analyze_entity_sentiment_template_fields] - @apply_defaults def __init__( self, *, @@ -237,7 +234,6 @@ class CloudNaturalLanguageAnalyzeSentimentOperator(BaseOperator): ) # [END natural_language_analyze_sentiment_template_fields] - @apply_defaults def __init__( self, *, @@ -313,7 +309,6 @@ class CloudNaturalLanguageClassifyTextOperator(BaseOperator): ) # [END natural_language_classify_text_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/pubsub.py b/airflow/providers/google/cloud/operators/pubsub.py index 23b545f8fb3ce..ebef1b0b480ad 100644 --- a/airflow/providers/google/cloud/operators/pubsub.py +++ b/airflow/providers/google/cloud/operators/pubsub.py @@ -32,7 +32,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.pubsub import PubSubHook -from airflow.utils.decorators import apply_defaults class PubSubCreateTopicOperator(BaseOperator): @@ -126,7 +125,6 @@ class PubSubCreateTopicOperator(BaseOperator): ui_color = '#0273d4' # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, @@ -346,7 +344,6 @@ class PubSubCreateSubscriptionOperator(BaseOperator): ui_color = '#0273d4' # pylint: disable=too-many-arguments, too-many-locals - @apply_defaults def __init__( self, *, @@ -526,7 +523,6 @@ class PubSubDeleteTopicOperator(BaseOperator): ] ui_color = '#cb4335' - @apply_defaults def __init__( self, *, @@ -659,7 +655,6 @@ class PubSubDeleteSubscriptionOperator(BaseOperator): ] ui_color = '#cb4335' - @apply_defaults def __init__( self, *, @@ -786,7 +781,6 @@ class PubSubPublishMessageOperator(BaseOperator): ] ui_color = '#0273d4' - @apply_defaults def __init__( self, *, @@ -892,7 +886,6 @@ class PubSubPullOperator(BaseOperator): 'impersonation_chain', ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/spanner.py b/airflow/providers/google/cloud/operators/spanner.py index fc657a2010d7c..19d904c58d7b2 100644 --- a/airflow/providers/google/cloud/operators/spanner.py +++ b/airflow/providers/google/cloud/operators/spanner.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.spanner import SpannerHook -from airflow.utils.decorators import apply_defaults class SpannerDeployInstanceOperator(BaseOperator): @@ -73,7 +72,6 @@ class SpannerDeployInstanceOperator(BaseOperator): ) # [END gcp_spanner_deploy_template_fields] - @apply_defaults def __init__( self, *, @@ -158,7 +156,6 @@ class SpannerDeleteInstanceOperator(BaseOperator): ) # [END gcp_spanner_delete_template_fields] - @apply_defaults def __init__( self, *, @@ -240,7 +237,6 @@ class SpannerQueryDatabaseInstanceOperator(BaseOperator): template_ext = ('.sql',) # [END gcp_spanner_query_template_fields] - @apply_defaults def __init__( self, *, @@ -350,7 +346,6 @@ class SpannerDeployDatabaseInstanceOperator(BaseOperator): template_ext = ('.sql',) # [END gcp_spanner_database_deploy_template_fields] - @apply_defaults def __init__( self, *, @@ -455,7 +450,6 @@ class SpannerUpdateDatabaseInstanceOperator(BaseOperator): template_ext = ('.sql',) # [END gcp_spanner_database_update_template_fields] - @apply_defaults def __init__( self, *, @@ -549,7 +543,6 @@ class SpannerDeleteDatabaseInstanceOperator(BaseOperator): ) # [END gcp_spanner_database_delete_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/speech_to_text.py b/airflow/providers/google/cloud/operators/speech_to_text.py index 3b5434ff114d9..2f3bf647122f4 100644 --- a/airflow/providers/google/cloud/operators/speech_to_text.py +++ b/airflow/providers/google/cloud/operators/speech_to_text.py @@ -25,7 +25,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.speech_to_text import CloudSpeechToTextHook, RecognitionAudio -from airflow.utils.decorators import apply_defaults class CloudSpeechToTextRecognizeSpeechOperator(BaseOperator): @@ -77,7 +76,6 @@ class CloudSpeechToTextRecognizeSpeechOperator(BaseOperator): ) # [END gcp_speech_to_text_synthesize_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/stackdriver.py b/airflow/providers/google/cloud/operators/stackdriver.py index 7289b1227bc5e..672cf38f27498 100644 --- a/airflow/providers/google/cloud/operators/stackdriver.py +++ b/airflow/providers/google/cloud/operators/stackdriver.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.stackdriver import StackdriverHook -from airflow.utils.decorators import apply_defaults class StackdriverListAlertPoliciesOperator(BaseOperator): @@ -93,7 +92,6 @@ class StackdriverListAlertPoliciesOperator(BaseOperator): ui_color = "#e5ffcc" # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, @@ -201,7 +199,6 @@ class StackdriverEnableAlertPoliciesOperator(BaseOperator): 'impersonation_chain', ) - @apply_defaults def __init__( self, *, @@ -292,7 +289,6 @@ class StackdriverDisableAlertPoliciesOperator(BaseOperator): 'impersonation_chain', ) - @apply_defaults def __init__( self, *, @@ -385,7 +381,6 @@ class StackdriverUpsertAlertOperator(BaseOperator): ui_color = "#e5ffcc" - @apply_defaults def __init__( self, *, @@ -474,7 +469,6 @@ class StackdriverDeleteAlertOperator(BaseOperator): ui_color = "#e5ffcc" - @apply_defaults def __init__( self, *, @@ -583,7 +577,6 @@ class StackdriverListNotificationChannelsOperator(BaseOperator): ui_color = "#e5ffcc" # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, @@ -692,7 +685,6 @@ class StackdriverEnableNotificationChannelsOperator(BaseOperator): ui_color = "#e5ffcc" - @apply_defaults def __init__( self, *, @@ -785,7 +777,6 @@ class StackdriverDisableNotificationChannelsOperator(BaseOperator): ui_color = "#e5ffcc" - @apply_defaults def __init__( self, *, @@ -880,7 +871,6 @@ class StackdriverUpsertNotificationChannelOperator(BaseOperator): ui_color = "#e5ffcc" - @apply_defaults def __init__( self, *, @@ -971,7 +961,6 @@ class StackdriverDeleteNotificationChannelOperator(BaseOperator): ui_color = "#e5ffcc" - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/tasks.py b/airflow/providers/google/cloud/operators/tasks.py index 2834b324c8aeb..1d187866e6a45 100644 --- a/airflow/providers/google/cloud/operators/tasks.py +++ b/airflow/providers/google/cloud/operators/tasks.py @@ -30,7 +30,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.tasks import CloudTasksHook -from airflow.utils.decorators import apply_defaults MetaData = Sequence[Tuple[str, str]] @@ -84,7 +83,6 @@ class CloudTasksQueueCreateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -193,7 +191,6 @@ class CloudTasksQueueUpdateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -282,7 +279,6 @@ class CloudTasksQueueGetOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -367,7 +363,6 @@ class CloudTasksQueuesListOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -451,7 +446,6 @@ class CloudTasksQueueDeleteOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -533,7 +527,6 @@ class CloudTasksQueuePurgeOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -616,7 +609,6 @@ class CloudTasksQueuePauseOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -699,7 +691,6 @@ class CloudTasksQueueResumeOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -793,7 +784,6 @@ class CloudTasksTaskCreateOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -891,7 +881,6 @@ class CloudTasksTaskGetOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -986,7 +975,6 @@ class CloudTasksTasksListOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1076,7 +1064,6 @@ class CloudTasksTaskDeleteOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -1167,7 +1154,6 @@ class CloudTasksTaskRunOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/text_to_speech.py b/airflow/providers/google/cloud/operators/text_to_speech.py index ab73c35822f6c..0740b8319cbe4 100644 --- a/airflow/providers/google/cloud/operators/text_to_speech.py +++ b/airflow/providers/google/cloud/operators/text_to_speech.py @@ -27,7 +27,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.cloud.hooks.text_to_speech import CloudTextToSpeechHook -from airflow.utils.decorators import apply_defaults class CloudTextToSpeechSynthesizeOperator(BaseOperator): @@ -88,7 +87,6 @@ class CloudTextToSpeechSynthesizeOperator(BaseOperator): ) # [END gcp_text_to_speech_synthesize_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/translate.py b/airflow/providers/google/cloud/operators/translate.py index d38bb7c537960..12c2aaf1a9fa3 100644 --- a/airflow/providers/google/cloud/operators/translate.py +++ b/airflow/providers/google/cloud/operators/translate.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.translate import CloudTranslateHook -from airflow.utils.decorators import apply_defaults class CloudTranslateTextOperator(BaseOperator): @@ -94,7 +93,6 @@ class CloudTranslateTextOperator(BaseOperator): ) # [END translate_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/translate_speech.py b/airflow/providers/google/cloud/operators/translate_speech.py index 1ec734bc10428..3b030a9b70b44 100644 --- a/airflow/providers/google/cloud/operators/translate_speech.py +++ b/airflow/providers/google/cloud/operators/translate_speech.py @@ -25,7 +25,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.speech_to_text import CloudSpeechToTextHook from airflow.providers.google.cloud.hooks.translate import CloudTranslateHook -from airflow.utils.decorators import apply_defaults class CloudTranslateSpeechOperator(BaseOperator): @@ -118,7 +117,6 @@ class CloudTranslateSpeechOperator(BaseOperator): ) # [END translate_speech_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/video_intelligence.py b/airflow/providers/google/cloud/operators/video_intelligence.py index fdb73cd1b0164..4fcf5c3937d54 100644 --- a/airflow/providers/google/cloud/operators/video_intelligence.py +++ b/airflow/providers/google/cloud/operators/video_intelligence.py @@ -25,7 +25,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.video_intelligence import CloudVideoIntelligenceHook -from airflow.utils.decorators import apply_defaults class CloudVideoIntelligenceDetectVideoLabelsOperator(BaseOperator): @@ -82,7 +81,6 @@ class CloudVideoIntelligenceDetectVideoLabelsOperator(BaseOperator): ) # [END gcp_video_intelligence_detect_labels_template_fields] - @apply_defaults def __init__( self, *, @@ -182,7 +180,6 @@ class CloudVideoIntelligenceDetectVideoExplicitContentOperator(BaseOperator): ) # [END gcp_video_intelligence_detect_explicit_content_template_fields] - @apply_defaults def __init__( self, *, @@ -282,7 +279,6 @@ class CloudVideoIntelligenceDetectVideoShotsOperator(BaseOperator): ) # [END gcp_video_intelligence_detect_video_shots_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/operators/vision.py b/airflow/providers/google/cloud/operators/vision.py index 73a47fb36706d..9203f23e8f265 100644 --- a/airflow/providers/google/cloud/operators/vision.py +++ b/airflow/providers/google/cloud/operators/vision.py @@ -33,7 +33,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.vision import CloudVisionHook -from airflow.utils.decorators import apply_defaults MetaData = Sequence[Tuple[str, str]] @@ -92,7 +91,6 @@ class CloudVisionCreateProductSetOperator(BaseOperator): ) # [END vision_productset_create_template_fields] - @apply_defaults def __init__( self, *, @@ -189,7 +187,6 @@ class CloudVisionGetProductSetOperator(BaseOperator): ) # [END vision_productset_get_template_fields] - @apply_defaults def __init__( self, *, @@ -295,7 +292,6 @@ class CloudVisionUpdateProductSetOperator(BaseOperator): ) # [END vision_productset_update_template_fields] - @apply_defaults def __init__( self, *, @@ -390,7 +386,6 @@ class CloudVisionDeleteProductSetOperator(BaseOperator): ) # [END vision_productset_delete_template_fields] - @apply_defaults def __init__( self, *, @@ -489,7 +484,6 @@ class CloudVisionCreateProductOperator(BaseOperator): ) # [END vision_product_create_template_fields] - @apply_defaults def __init__( self, *, @@ -589,7 +583,6 @@ class CloudVisionGetProductOperator(BaseOperator): ) # [END vision_product_get_template_fields] - @apply_defaults def __init__( self, *, @@ -706,7 +699,6 @@ class CloudVisionUpdateProductOperator(BaseOperator): ) # [END vision_product_update_template_fields] - @apply_defaults def __init__( self, *, @@ -806,7 +798,6 @@ class CloudVisionDeleteProductOperator(BaseOperator): ) # [END vision_product_delete_template_fields] - @apply_defaults def __init__( self, *, @@ -886,7 +877,6 @@ class CloudVisionImageAnnotateOperator(BaseOperator): ) # [END vision_annotate_image_template_fields] - @apply_defaults def __init__( self, *, @@ -980,7 +970,6 @@ class CloudVisionCreateReferenceImageOperator(BaseOperator): ) # [END vision_reference_image_create_template_fields] - @apply_defaults def __init__( self, *, @@ -1086,7 +1075,6 @@ class CloudVisionDeleteReferenceImageOperator(BaseOperator): ) # [END vision_reference_image_create_template_fields] - @apply_defaults def __init__( self, *, @@ -1185,7 +1173,6 @@ class CloudVisionAddProductToProductSetOperator(BaseOperator): ) # [END vision_add_product_to_product_set_template_fields] - @apply_defaults def __init__( self, *, @@ -1278,7 +1265,6 @@ class CloudVisionRemoveProductFromProductSetOperator(BaseOperator): ) # [END vision_remove_product_from_product_set_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/sensors/bigquery.py b/airflow/providers/google/cloud/sensors/bigquery.py index 9213bae2139c0..ab1eea5253cb7 100644 --- a/airflow/providers/google/cloud/sensors/bigquery.py +++ b/airflow/providers/google/cloud/sensors/bigquery.py @@ -20,7 +20,6 @@ from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class BigQueryTableExistenceSensor(BaseSensorOperator): @@ -62,7 +61,6 @@ class BigQueryTableExistenceSensor(BaseSensorOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, @@ -138,7 +136,6 @@ class BigQueryTablePartitionExistenceSensor(BaseSensorOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/sensors/bigquery_dts.py b/airflow/providers/google/cloud/sensors/bigquery_dts.py index 49e124c842f26..4b92cb4ee9cea 100644 --- a/airflow/providers/google/cloud/sensors/bigquery_dts.py +++ b/airflow/providers/google/cloud/sensors/bigquery_dts.py @@ -23,7 +23,6 @@ from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class BigQueryDataTransferServiceTransferRunSensor(BaseSensorOperator): @@ -75,7 +74,6 @@ class BigQueryDataTransferServiceTransferRunSensor(BaseSensorOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/sensors/bigtable.py b/airflow/providers/google/cloud/sensors/bigtable.py index e9190f4255b39..d292eb1ba77d1 100644 --- a/airflow/providers/google/cloud/sensors/bigtable.py +++ b/airflow/providers/google/cloud/sensors/bigtable.py @@ -25,7 +25,6 @@ from airflow.providers.google.cloud.hooks.bigtable import BigtableHook from airflow.providers.google.cloud.operators.bigtable import BigtableValidationMixin from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class BigtableTableReplicationCompletedSensor(BaseSensorOperator, BigtableValidationMixin): @@ -65,7 +64,6 @@ class BigtableTableReplicationCompletedSensor(BaseSensorOperator, BigtableValida 'impersonation_chain', ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py b/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py index c5c8036a31ae7..2c6252e68e750 100644 --- a/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +++ b/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py @@ -25,7 +25,6 @@ CloudDataTransferServiceHook, ) from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class CloudDataTransferServiceJobStatusSensor(BaseSensorOperator): @@ -67,7 +66,6 @@ class CloudDataTransferServiceJobStatusSensor(BaseSensorOperator): ) # [END gcp_transfer_job_sensor_template_fields] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/sensors/dataflow.py b/airflow/providers/google/cloud/sensors/dataflow.py index 58026e380e255..029fcebc4fb7a 100644 --- a/airflow/providers/google/cloud/sensors/dataflow.py +++ b/airflow/providers/google/cloud/sensors/dataflow.py @@ -25,7 +25,6 @@ DataflowJobStatus, ) from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class DataflowJobStatusSensor(BaseSensorOperator): @@ -68,7 +67,6 @@ class DataflowJobStatusSensor(BaseSensorOperator): template_fields = ['job_id'] - @apply_defaults def __init__( self, *, @@ -163,7 +161,6 @@ class DataflowJobMetricsSensor(BaseSensorOperator): template_fields = ['job_id'] - @apply_defaults def __init__( self, *, @@ -257,7 +254,6 @@ class DataflowJobMessagesSensor(BaseSensorOperator): template_fields = ['job_id'] - @apply_defaults def __init__( self, *, @@ -351,7 +347,6 @@ class DataflowJobAutoScalingEventsSensor(BaseSensorOperator): template_fields = ['job_id'] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/sensors/dataproc.py b/airflow/providers/google/cloud/sensors/dataproc.py index 93656df257173..b4a05472b7060 100644 --- a/airflow/providers/google/cloud/sensors/dataproc.py +++ b/airflow/providers/google/cloud/sensors/dataproc.py @@ -23,7 +23,6 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.dataproc import DataprocHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class DataprocJobSensor(BaseSensorOperator): @@ -44,7 +43,6 @@ class DataprocJobSensor(BaseSensorOperator): template_fields = ('project_id', 'location', 'dataproc_job_id') ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/sensors/gcs.py b/airflow/providers/google/cloud/sensors/gcs.py index 70b7fddb25f3d..69f4fd89fee5e 100644 --- a/airflow/providers/google/cloud/sensors/gcs.py +++ b/airflow/providers/google/cloud/sensors/gcs.py @@ -25,7 +25,6 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.sensors.base import BaseSensorOperator, poke_mode_only -from airflow.utils.decorators import apply_defaults class GCSObjectExistenceSensor(BaseSensorOperator): @@ -62,7 +61,6 @@ class GCSObjectExistenceSensor(BaseSensorOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, @@ -138,7 +136,6 @@ class GCSObjectUpdateSensor(BaseSensorOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, bucket: str, @@ -206,7 +203,6 @@ class GCSObjectsWithPrefixExistenceSensor(BaseSensorOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, bucket: str, @@ -317,7 +313,6 @@ class GCSUploadSessionCompleteSensor(BaseSensorOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, bucket: str, diff --git a/airflow/providers/google/cloud/sensors/pubsub.py b/airflow/providers/google/cloud/sensors/pubsub.py index ff1f811c5cf26..4945e063b5c8c 100644 --- a/airflow/providers/google/cloud/sensors/pubsub.py +++ b/airflow/providers/google/cloud/sensors/pubsub.py @@ -23,7 +23,6 @@ from airflow.providers.google.cloud.hooks.pubsub import PubSubHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class PubSubPullSensor(BaseSensorOperator): @@ -102,7 +101,6 @@ class PubSubPullSensor(BaseSensorOperator): ] ui_color = '#ff7f50' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/adls_to_gcs.py b/airflow/providers/google/cloud/transfers/adls_to_gcs.py index d75a0ae86740f..890284640254e 100644 --- a/airflow/providers/google/cloud/transfers/adls_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/adls_to_gcs.py @@ -27,7 +27,6 @@ from airflow.providers.google.cloud.hooks.gcs import GCSHook, _parse_gcs_url from airflow.providers.microsoft.azure.hooks.azure_data_lake import AzureDataLakeHook from airflow.providers.microsoft.azure.operators.adls_list import AzureDataLakeStorageListOperator -from airflow.utils.decorators import apply_defaults class ADLSToGCSOperator(AzureDataLakeStorageListOperator): @@ -111,7 +110,6 @@ class ADLSToGCSOperator(AzureDataLakeStorageListOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py b/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py index 3da7ba40547d2..528f735a61ef1 100644 --- a/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook, _parse_gcs_url, gcs_object_is_directory from airflow.providers.microsoft.azure.hooks.azure_fileshare import AzureFileShareHook -from airflow.utils.decorators import apply_defaults class AzureFileShareToGCSOperator(BaseOperator): @@ -76,7 +75,6 @@ class AzureFileShareToGCSOperator(BaseOperator): 'dest_gcs', ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py b/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py index c703abca47232..c0e3b6a6b11aa 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook -from airflow.utils.decorators import apply_defaults class BigQueryToBigQueryOperator(BaseOperator): @@ -86,7 +85,6 @@ class BigQueryToBigQueryOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#e6f0e4' - @apply_defaults def __init__( self, *, # pylint: disable=too-many-arguments diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py b/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py index 89ddb8621652d..e0ead8cd79b83 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook -from airflow.utils.decorators import apply_defaults class BigQueryToGCSOperator(BaseOperator): @@ -86,7 +85,6 @@ class BigQueryToGCSOperator(BaseOperator): template_ext = () ui_color = '#e4e6f0' - @apply_defaults def __init__( self, *, # pylint: disable=too-many-arguments diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py b/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py index 9665634f0acc6..6acf0828345fd 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.providers.mysql.hooks.mysql import MySqlHook -from airflow.utils.decorators import apply_defaults class BigQueryToMySqlOperator(BaseOperator): @@ -87,7 +86,6 @@ class BigQueryToMySqlOperator(BaseOperator): 'impersonation_chain', ) - @apply_defaults def __init__( self, *, # pylint: disable=too-many-arguments diff --git a/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py b/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py index bc348db9f7cf9..e7d0bf4585220 100644 --- a/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py @@ -35,7 +35,6 @@ from airflow.models import BaseOperator from airflow.providers.apache.cassandra.hooks.cassandra import CassandraHook from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults class CassandraToGCSOperator(BaseOperator): @@ -97,7 +96,6 @@ class CassandraToGCSOperator(BaseOperator): template_ext = ('.cql',) ui_color = '#a0e08c' - @apply_defaults def __init__( self, *, # pylint: disable=too-many-arguments diff --git a/airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py b/airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py index f4fec21d284d0..ed13fd6021455 100644 --- a/airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.facebook.ads.hooks.ads import FacebookAdsReportingHook from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults class FacebookAdsReportToGcsOperator(BaseOperator): @@ -80,7 +79,6 @@ class FacebookAdsReportToGcsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index 60cf505a67abe..b9251100ed999 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -24,7 +24,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults # pylint: disable=too-many-instance-attributes @@ -135,7 +134,7 @@ class GCSToBigQueryOperator(BaseOperator): :type cluster_fields: list[str] :param autodetect: [Optional] Indicates if we should automatically infer the options and schema for CSV and JSON sources. (Default: ``True``). - Parameter must be setted to True if 'schema_fields' and 'schema_object' are undefined. + Parameter must be set to True if 'schema_fields' and 'schema_object' are undefined. It is suggested to set to True if table are create outside of Airflow. :type autodetect: bool :param encryption_configuration: [Optional] Custom encryption configuration (e.g., Cloud KMS keys). @@ -174,7 +173,6 @@ class GCSToBigQueryOperator(BaseOperator): ui_color = '#f0eee4' # pylint: disable=too-many-locals,too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py index b40fc3e6bf0be..c7af735a7d3b6 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py @@ -22,7 +22,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults WILDCARD = '*' @@ -185,7 +184,6 @@ class GCSToGCSOperator(BaseOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, # pylint: disable=too-many-arguments diff --git a/airflow/providers/google/cloud/transfers/gcs_to_local.py b/airflow/providers/google/cloud/transfers/gcs_to_local.py index 12a61eff8b806..a76502691aaef 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_local.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_local.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.models.xcom import MAX_XCOM_SIZE from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.sensors.base import apply_defaults class GCSToLocalFilesystemOperator(BaseOperator): @@ -82,7 +81,6 @@ class GCSToLocalFilesystemOperator(BaseOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/gcs_to_sftp.py b/airflow/providers/google/cloud/transfers/gcs_to_sftp.py index 2f5a32d868ec2..11edc54051e1e 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_sftp.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_sftp.py @@ -24,7 +24,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.sftp.hooks.sftp import SFTPHook -from airflow.utils.decorators import apply_defaults WILDCARD = "*" @@ -111,7 +110,6 @@ class GCSToSFTPOperator(BaseOperator): ui_color = "#f0eee4" # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/gdrive_to_gcs.py b/airflow/providers/google/cloud/transfers/gdrive_to_gcs.py index 88c40083054d0..4ed72933ed66d 100644 --- a/airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/gdrive_to_gcs.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.suite.hooks.drive import GoogleDriveHook -from airflow.utils.decorators import apply_defaults class GoogleDriveToGCSOperator(BaseOperator): @@ -76,7 +75,6 @@ class GoogleDriveToGCSOperator(BaseOperator): "impersonation_chain", ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/gdrive_to_local.py b/airflow/providers/google/cloud/transfers/gdrive_to_local.py index b03c8dc5ac754..113a389c6fbf2 100644 --- a/airflow/providers/google/cloud/transfers/gdrive_to_local.py +++ b/airflow/providers/google/cloud/transfers/gdrive_to_local.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.google.suite.hooks.drive import GoogleDriveHook -from airflow.utils.decorators import apply_defaults class GoogleDriveToLocalOperator(BaseOperator): @@ -61,7 +60,6 @@ class GoogleDriveToLocalOperator(BaseOperator): "impersonation_chain", ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/local_to_gcs.py b/airflow/providers/google/cloud/transfers/local_to_gcs.py index 0a8b78cf5d40e..688832d147f8d 100644 --- a/airflow/providers/google/cloud/transfers/local_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/local_to_gcs.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults class LocalFilesystemToGCSOperator(BaseOperator): @@ -73,7 +72,6 @@ class LocalFilesystemToGCSOperator(BaseOperator): 'impersonation_chain', ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/mssql_to_gcs.py b/airflow/providers/google/cloud/transfers/mssql_to_gcs.py index 0157b9a395c00..6a3289b205d5d 100644 --- a/airflow/providers/google/cloud/transfers/mssql_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/mssql_to_gcs.py @@ -22,7 +22,6 @@ from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator from airflow.providers.microsoft.mssql.hooks.mssql import MsSqlHook -from airflow.utils.decorators import apply_defaults class MSSQLToGCSOperator(BaseSQLToGCSOperator): @@ -53,7 +52,6 @@ class MSSQLToGCSOperator(BaseSQLToGCSOperator): type_map = {3: 'INTEGER', 4: 'TIMESTAMP', 5: 'NUMERIC'} - @apply_defaults def __init__(self, *, mssql_conn_id='mssql_default', **kwargs): super().__init__(**kwargs) self.mssql_conn_id = mssql_conn_id diff --git a/airflow/providers/google/cloud/transfers/mysql_to_gcs.py b/airflow/providers/google/cloud/transfers/mysql_to_gcs.py index 8ebe4b810d0b9..6f5c4c20d6df1 100644 --- a/airflow/providers/google/cloud/transfers/mysql_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/mysql_to_gcs.py @@ -27,7 +27,6 @@ from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator from airflow.providers.mysql.hooks.mysql import MySqlHook -from airflow.utils.decorators import apply_defaults class MySQLToGCSOperator(BaseSQLToGCSOperator): @@ -65,7 +64,6 @@ class MySQLToGCSOperator(BaseSQLToGCSOperator): FIELD_TYPE.YEAR: 'INTEGER', } - @apply_defaults def __init__(self, *, mysql_conn_id='mysql_default', ensure_utc=False, **kwargs): super().__init__(**kwargs) self.mysql_conn_id = mysql_conn_id diff --git a/airflow/providers/google/cloud/transfers/oracle_to_gcs.py b/airflow/providers/google/cloud/transfers/oracle_to_gcs.py index 8c8af7143627b..d4ede0e41bef3 100644 --- a/airflow/providers/google/cloud/transfers/oracle_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/oracle_to_gcs.py @@ -26,7 +26,6 @@ from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator from airflow.providers.oracle.hooks.oracle import OracleHook -from airflow.utils.decorators import apply_defaults class OracleToGCSOperator(BaseSQLToGCSOperator): @@ -36,7 +35,8 @@ class OracleToGCSOperator(BaseSQLToGCSOperator): For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:OracleToGCSOperator` - :param oracle_conn_id: Reference to a specific Oracle hook. + :param oracle_conn_id: Reference to a specific + :ref:`Oracle hook `. :type oracle_conn_id: str :param ensure_utc: Ensure TIMESTAMP columns exported as UTC. If set to `False`, TIMESTAMP columns will be exported using the Oracle server's @@ -58,7 +58,6 @@ class OracleToGCSOperator(BaseSQLToGCSOperator): cx_Oracle.DB_TYPE_TIMESTAMP_TZ: 'TIMESTAMP', } - @apply_defaults def __init__(self, *, oracle_conn_id='oracle_default', ensure_utc=False, **kwargs): super().__init__(**kwargs) self.ensure_utc = ensure_utc diff --git a/airflow/providers/google/cloud/transfers/postgres_to_gcs.py b/airflow/providers/google/cloud/transfers/postgres_to_gcs.py index 68405049b70d9..b5606c6697e4a 100644 --- a/airflow/providers/google/cloud/transfers/postgres_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/postgres_to_gcs.py @@ -28,7 +28,6 @@ from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator from airflow.providers.postgres.hooks.postgres import PostgresHook -from airflow.utils.decorators import apply_defaults class _PostgresServerSideCursorDecorator: @@ -97,7 +96,6 @@ class PostgresToGCSOperator(BaseSQLToGCSOperator): 1700: 'FLOAT', } - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/presto_to_gcs.py b/airflow/providers/google/cloud/transfers/presto_to_gcs.py index 70a4062b0def0..1903b6b2ff96a 100644 --- a/airflow/providers/google/cloud/transfers/presto_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/presto_to_gcs.py @@ -22,7 +22,6 @@ from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator from airflow.providers.presto.hooks.presto import PrestoHook -from airflow.utils.decorators import apply_defaults class _PrestoToGCSPrestoCursorAdapter: @@ -175,7 +174,6 @@ class PrestoToGCSOperator(BaseSQLToGCSOperator): "UUID": "STRING", } - @apply_defaults def __init__(self, *, presto_conn_id: str = "presto_default", **kwargs): super().__init__(**kwargs) self.presto_conn_id = presto_conn_id diff --git a/airflow/providers/google/cloud/transfers/s3_to_gcs.py b/airflow/providers/google/cloud/transfers/s3_to_gcs.py index d0db35f460851..81cd6c48b4c9c 100644 --- a/airflow/providers/google/cloud/transfers/s3_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/s3_to_gcs.py @@ -23,7 +23,6 @@ from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.amazon.aws.operators.s3_list import S3ListOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook, _parse_gcs_url, gcs_object_is_directory -from airflow.utils.decorators import apply_defaults class S3ToGCSOperator(S3ListOperator): @@ -111,7 +110,6 @@ class S3ToGCSOperator(S3ListOperator): ui_color = '#e09411' # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/sftp_to_gcs.py b/airflow/providers/google/cloud/transfers/sftp_to_gcs.py index 2f54600ae67ca..35d0741ae3004 100644 --- a/airflow/providers/google/cloud/transfers/sftp_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/sftp_to_gcs.py @@ -24,7 +24,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.sftp.hooks.sftp import SFTPHook -from airflow.utils.decorators import apply_defaults WILDCARD = "*" @@ -86,7 +85,6 @@ class SFTPToGCSOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/sheets_to_gcs.py b/airflow/providers/google/cloud/transfers/sheets_to_gcs.py index 13edd6ca52260..a668ebfaddf29 100644 --- a/airflow/providers/google/cloud/transfers/sheets_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/sheets_to_gcs.py @@ -22,7 +22,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.suite.hooks.sheets import GSheetsHook -from airflow.utils.decorators import apply_defaults class GoogleSheetsToGCSOperator(BaseOperator): @@ -69,7 +68,6 @@ class GoogleSheetsToGCSOperator(BaseOperator): "impersonation_chain", ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/cloud/transfers/sql_to_gcs.py b/airflow/providers/google/cloud/transfers/sql_to_gcs.py index e62364a073851..975a845d62a02 100644 --- a/airflow/providers/google/cloud/transfers/sql_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/sql_to_gcs.py @@ -28,11 +28,12 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.utils.decorators import apply_defaults class BaseSQLToGCSOperator(BaseOperator): """ + Copy data from SQL to Google Cloud Storage in JSON or CSV format. + :param sql: The SQL to execute. :type sql: str :param bucket: The bucket to upload to. @@ -99,7 +100,6 @@ class BaseSQLToGCSOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#a0e08c' - @apply_defaults def __init__( self, *, # pylint: disable=too-many-arguments diff --git a/airflow/providers/google/cloud/transfers/trino_to_gcs.py b/airflow/providers/google/cloud/transfers/trino_to_gcs.py index e2f2306bc80cc..bccf49346e803 100644 --- a/airflow/providers/google/cloud/transfers/trino_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/trino_to_gcs.py @@ -22,7 +22,6 @@ from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator from airflow.providers.trino.hooks.trino import TrinoHook -from airflow.utils.decorators import apply_defaults class _TrinoToGCSTrinoCursorAdapter: @@ -175,7 +174,6 @@ class TrinoToGCSOperator(BaseSQLToGCSOperator): "UUID": "STRING", } - @apply_defaults def __init__(self, *, trino_conn_id: str = "trino_default", **kwargs): super().__init__(**kwargs) self.trino_conn_id = trino_conn_id diff --git a/airflow/providers/google/firebase/example_dags/example_firestore.py b/airflow/providers/google/firebase/example_dags/example_firestore.py index 3bdd1712d72ef..df0bb3a0b50ea 100644 --- a/airflow/providers/google/firebase/example_dags/example_firestore.py +++ b/airflow/providers/google/firebase/example_dags/example_firestore.py @@ -59,7 +59,7 @@ GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-gcp-project") FIRESTORE_PROJECT_ID = os.environ.get("G_FIRESTORE_PROJECT_ID", "example-firebase-project") -EXPORT_DESTINATION_URL = os.environ.get("GCP_FIRESTORE_ARCHIVE_URL", "gs://airflow-firestore/namespace/") +EXPORT_DESTINATION_URL = os.environ.get("GCP_FIRESTORE_ARCHIVE_URL", "gs://INVALID BUCKET NAME/namespace/") BUCKET_NAME = urlparse(EXPORT_DESTINATION_URL).hostname EXPORT_PREFIX = urlparse(EXPORT_DESTINATION_URL).path diff --git a/airflow/providers/google/firebase/operators/firestore.py b/airflow/providers/google/firebase/operators/firestore.py index 6778fbe7aea7d..a49cac8a4e0b9 100644 --- a/airflow/providers/google/firebase/operators/firestore.py +++ b/airflow/providers/google/firebase/operators/firestore.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.google.firebase.hooks.firestore import CloudFirestoreHook -from airflow.utils.decorators import apply_defaults class CloudFirestoreExportDatabaseOperator(BaseOperator): @@ -63,7 +62,6 @@ class CloudFirestoreExportDatabaseOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/leveldb/operators/leveldb.py b/airflow/providers/google/leveldb/operators/leveldb.py index d1e941852fffc..6ed7660b9af56 100644 --- a/airflow/providers/google/leveldb/operators/leveldb.py +++ b/airflow/providers/google/leveldb/operators/leveldb.py @@ -18,7 +18,6 @@ from airflow.models import BaseOperator from airflow.providers.google.leveldb.hooks.leveldb import LevelDBHook -from airflow.utils.decorators import apply_defaults class LevelDBOperator(BaseOperator): @@ -49,7 +48,6 @@ class LevelDBOperator(BaseOperator): :type create_db_extra_options: Optional[Dict[str, Any]] """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/marketing_platform/example_dags/example_display_video.py b/airflow/providers/google/marketing_platform/example_dags/example_display_video.py index da98123f34be0..cab062595682b 100644 --- a/airflow/providers/google/marketing_platform/example_dags/example_display_video.py +++ b/airflow/providers/google/marketing_platform/example_dags/example_display_video.py @@ -41,7 +41,7 @@ from airflow.utils import dates # [START howto_display_video_env_variables] -BUCKET = os.environ.get("GMP_DISPLAY_VIDEO_BUCKET", "gs://test-display-video-bucket") +BUCKET = os.environ.get("GMP_DISPLAY_VIDEO_BUCKET", "gs://INVALID BUCKET NAME") ADVERTISER_ID = os.environ.get("GMP_ADVERTISER_ID", 1234567) OBJECT_NAME = os.environ.get("GMP_OBJECT_NAME", "files/report.csv") PATH_TO_UPLOAD_FILE = os.environ.get("GCP_GCS_PATH_TO_UPLOAD_FILE", "test-gcs-example.txt") diff --git a/airflow/providers/google/marketing_platform/operators/analytics.py b/airflow/providers/google/marketing_platform/operators/analytics.py index 2a1f6a14c6f23..68294014d832d 100644 --- a/airflow/providers/google/marketing_platform/operators/analytics.py +++ b/airflow/providers/google/marketing_platform/operators/analytics.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.marketing_platform.hooks.analytics import GoogleAnalyticsHook -from airflow.utils.decorators import apply_defaults class GoogleAnalyticsListAccountsOperator(BaseOperator): @@ -61,7 +60,6 @@ class GoogleAnalyticsListAccountsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -124,7 +122,6 @@ class GoogleAnalyticsGetAdsLinkOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -194,7 +191,6 @@ class GoogleAnalyticsRetrieveAdsLinksListOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -270,7 +266,6 @@ class GoogleAnalyticsDataImportUploadOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/marketing_platform/operators/campaign_manager.py b/airflow/providers/google/marketing_platform/operators/campaign_manager.py index 9139c997f28fc..6a77d9fb3bd07 100644 --- a/airflow/providers/google/marketing_platform/operators/campaign_manager.py +++ b/airflow/providers/google/marketing_platform/operators/campaign_manager.py @@ -27,7 +27,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.marketing_platform.hooks.campaign_manager import GoogleCampaignManagerHook -from airflow.utils.decorators import apply_defaults class GoogleCampaignManagerDeleteReportOperator(BaseOperator): @@ -77,7 +76,6 @@ class GoogleCampaignManagerDeleteReportOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -183,7 +181,6 @@ class GoogleCampaignManagerDownloadReportOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, @@ -314,7 +311,6 @@ class GoogleCampaignManagerInsertReportOperator(BaseOperator): template_ext = (".json",) - @apply_defaults def __init__( self, *, @@ -402,7 +398,6 @@ class GoogleCampaignManagerRunReportOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -499,7 +494,6 @@ class GoogleCampaignManagerBatchInsertConversionsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -601,7 +595,6 @@ class GoogleCampaignManagerBatchUpdateConversionsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/marketing_platform/operators/display_video.py b/airflow/providers/google/marketing_platform/operators/display_video.py index 08eb7687b4fbf..9f1f9f9c8d968 100644 --- a/airflow/providers/google/marketing_platform/operators/display_video.py +++ b/airflow/providers/google/marketing_platform/operators/display_video.py @@ -28,7 +28,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.marketing_platform.hooks.display_video import GoogleDisplayVideo360Hook -from airflow.utils.decorators import apply_defaults class GoogleDisplayVideo360CreateReportOperator(BaseOperator): @@ -71,7 +70,6 @@ class GoogleDisplayVideo360CreateReportOperator(BaseOperator): ) template_ext = (".json",) - @apply_defaults def __init__( self, *, @@ -150,7 +148,6 @@ class GoogleDisplayVideo360DeleteReportOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -245,7 +242,6 @@ class GoogleDisplayVideo360DownloadReportOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -370,7 +366,6 @@ class GoogleDisplayVideo360RunReportOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -430,7 +425,6 @@ class GoogleDisplayVideo360DownloadLineItemsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -514,7 +508,6 @@ class GoogleDisplayVideo360UploadLineItemsOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -606,7 +599,6 @@ class GoogleDisplayVideo360CreateSDFDownloadTaskOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, @@ -686,7 +678,6 @@ class GoogleDisplayVideo360SDFtoGCSOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/marketing_platform/operators/search_ads.py b/airflow/providers/google/marketing_platform/operators/search_ads.py index e8984dcbb93cb..77f790f84b290 100644 --- a/airflow/providers/google/marketing_platform/operators/search_ads.py +++ b/airflow/providers/google/marketing_platform/operators/search_ads.py @@ -24,7 +24,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.marketing_platform.hooks.search_ads import GoogleSearchAdsHook -from airflow.utils.decorators import apply_defaults class GoogleSearchAdsInsertReportOperator(BaseOperator): @@ -66,7 +65,6 @@ class GoogleSearchAdsInsertReportOperator(BaseOperator): ) template_ext = (".json",) - @apply_defaults def __init__( self, *, @@ -152,7 +150,6 @@ class GoogleSearchAdsDownloadReportOperator(BaseOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/marketing_platform/sensors/campaign_manager.py b/airflow/providers/google/marketing_platform/sensors/campaign_manager.py index 7bdddefebbc60..686481787fe03 100644 --- a/airflow/providers/google/marketing_platform/sensors/campaign_manager.py +++ b/airflow/providers/google/marketing_platform/sensors/campaign_manager.py @@ -20,7 +20,6 @@ from airflow.providers.google.marketing_platform.hooks.campaign_manager import GoogleCampaignManagerHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class GoogleCampaignManagerReportSensor(BaseSensorOperator): @@ -78,7 +77,6 @@ def poke(self, context: Dict) -> bool: self.log.info("Report status: %s", response["status"]) return response["status"] != "PROCESSING" - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/marketing_platform/sensors/search_ads.py b/airflow/providers/google/marketing_platform/sensors/search_ads.py index fb31a91310505..4dc170594397e 100644 --- a/airflow/providers/google/marketing_platform/sensors/search_ads.py +++ b/airflow/providers/google/marketing_platform/sensors/search_ads.py @@ -20,7 +20,6 @@ from airflow.providers.google.marketing_platform.hooks.search_ads import GoogleSearchAdsHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class GoogleSearchAdsReportSensor(BaseSensorOperator): @@ -61,7 +60,6 @@ class GoogleSearchAdsReportSensor(BaseSensorOperator): "impersonation_chain", ) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 644f093be2297..87ba4ae1a8608 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -743,6 +743,8 @@ extra-links: - airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleLink - airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink - airflow.providers.google.cloud.operators.mlengine.AIPlatformConsoleLink + - airflow.providers.google.cloud.operators.dataproc.DataprocJobLink + - airflow.providers.google.cloud.operators.dataproc.DataprocClusterLink additional-extras: apache.beam: apache-beam[gcp] diff --git a/airflow/providers/google/suite/operators/sheets.py b/airflow/providers/google/suite/operators/sheets.py index 007ff439be06c..642f4f943d1cb 100644 --- a/airflow/providers/google/suite/operators/sheets.py +++ b/airflow/providers/google/suite/operators/sheets.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.google.suite.hooks.sheets import GSheetsHook -from airflow.utils.decorators import apply_defaults class GoogleSheetsCreateSpreadsheetOperator(BaseOperator): @@ -55,7 +54,6 @@ class GoogleSheetsCreateSpreadsheetOperator(BaseOperator): "impersonation_chain", ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/suite/sensors/drive.py b/airflow/providers/google/suite/sensors/drive.py index e92015a254e48..02eb97b61e5f9 100644 --- a/airflow/providers/google/suite/sensors/drive.py +++ b/airflow/providers/google/suite/sensors/drive.py @@ -21,7 +21,6 @@ from airflow.providers.google.suite.hooks.drive import GoogleDriveHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class GoogleDriveFileExistenceSensor(BaseSensorOperator): @@ -60,7 +59,6 @@ class GoogleDriveFileExistenceSensor(BaseSensorOperator): ) ui_color = '#f0eee4' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/suite/transfers/gcs_to_gdrive.py b/airflow/providers/google/suite/transfers/gcs_to_gdrive.py index 66d1538f21e38..b75611dd94cda 100644 --- a/airflow/providers/google/suite/transfers/gcs_to_gdrive.py +++ b/airflow/providers/google/suite/transfers/gcs_to_gdrive.py @@ -23,7 +23,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.suite.hooks.drive import GoogleDriveHook -from airflow.utils.decorators import apply_defaults WILDCARD = "*" @@ -89,7 +88,6 @@ class GCSToGoogleDriveOperator(BaseOperator): ) ui_color = "#f0eee4" - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/google/suite/transfers/gcs_to_sheets.py b/airflow/providers/google/suite/transfers/gcs_to_sheets.py index b74b6e319a063..92c3cfa29b29a 100644 --- a/airflow/providers/google/suite/transfers/gcs_to_sheets.py +++ b/airflow/providers/google/suite/transfers/gcs_to_sheets.py @@ -22,7 +22,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.suite.hooks.sheets import GSheetsHook -from airflow.utils.decorators import apply_defaults class GCSToGoogleSheetsOperator(BaseOperator): @@ -66,7 +65,6 @@ class GCSToGoogleSheetsOperator(BaseOperator): "impersonation_chain", ] - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/grpc/operators/grpc.py b/airflow/providers/grpc/operators/grpc.py index a31b388063cf6..0a5d111fb82db 100644 --- a/airflow/providers/grpc/operators/grpc.py +++ b/airflow/providers/grpc/operators/grpc.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.grpc.hooks.grpc import GrpcHook -from airflow.utils.decorators import apply_defaults class GrpcOperator(BaseOperator): @@ -53,7 +52,6 @@ class GrpcOperator(BaseOperator): template_fields = ('stub_class', 'call_func', 'data') template_fields_renderers = {"data": "py"} - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 542a620659f94..6cf5f8ebd6762 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -176,16 +176,23 @@ def run_and_check( """ extra_options = extra_options or {} + settings = session.merge_environment_settings( + prepped_request.url, + proxies=extra_options.get("proxies", {}), + stream=extra_options.get("stream", False), + verify=extra_options.get("verify"), + cert=extra_options.get("cert"), + ) + + # Send the request. + send_kwargs = { + "timeout": extra_options.get("timeout"), + "allow_redirects": extra_options.get("allow_redirects", True), + } + send_kwargs.update(settings) + try: - response = session.send( - prepped_request, - stream=extra_options.get("stream", False), - verify=extra_options.get("verify", True), - proxies=extra_options.get("proxies", {}), - cert=extra_options.get("cert"), - timeout=extra_options.get("timeout"), - allow_redirects=extra_options.get("allow_redirects", True), - ) + response = session.send(prepped_request, **send_kwargs) if extra_options.get('check_response', True): self.check_response(response) diff --git a/airflow/providers/http/operators/http.py b/airflow/providers/http/operators/http.py index 4258da084ed9f..b6295185d8ba8 100644 --- a/airflow/providers/http/operators/http.py +++ b/airflow/providers/http/operators/http.py @@ -15,12 +15,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, Optional, Type + +from requests.auth import AuthBase, HTTPBasicAuth from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.http.hooks.http import HttpHook -from airflow.utils.decorators import apply_defaults class SimpleHttpOperator(BaseOperator): @@ -60,6 +61,8 @@ class SimpleHttpOperator(BaseOperator): depends on the option that's being modified. :param log_response: Log the response (default: False) :type log_response: bool + :param auth_type: The auth type for the service + :type auth_type: AuthBase of python requests lib """ template_fields = [ @@ -71,7 +74,6 @@ class SimpleHttpOperator(BaseOperator): template_ext = () ui_color = '#f4a460' - @apply_defaults def __init__( self, *, @@ -84,6 +86,7 @@ def __init__( extra_options: Optional[Dict[str, Any]] = None, http_conn_id: str = 'http_default', log_response: bool = False, + auth_type: Type[AuthBase] = HTTPBasicAuth, **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -96,13 +99,14 @@ def __init__( self.response_filter = response_filter self.extra_options = extra_options or {} self.log_response = log_response + self.auth_type = auth_type if kwargs.get('xcom_push') is not None: raise AirflowException("'xcom_push' was deprecated, use 'BaseOperator.do_xcom_push' instead") def execute(self, context: Dict[str, Any]) -> Any: from airflow.utils.operator_helpers import make_kwargs_callable - http = HttpHook(self.method, http_conn_id=self.http_conn_id) + http = HttpHook(self.method, http_conn_id=self.http_conn_id, auth_type=self.auth_type) self.log.info("Calling HTTP method") diff --git a/airflow/providers/http/sensors/http.py b/airflow/providers/http/sensors/http.py index 917bbecb39a33..6ef55ea5a5641 100644 --- a/airflow/providers/http/sensors/http.py +++ b/airflow/providers/http/sensors/http.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.providers.http.hooks.http import HttpHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class HttpSensor(BaseSensorOperator): @@ -74,7 +73,6 @@ def response_check(response, task_instance): template_fields = ('endpoint', 'request_params', 'headers') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/imap/sensors/imap_attachment.py b/airflow/providers/imap/sensors/imap_attachment.py index a4dcb0408f466..9ad128d762399 100644 --- a/airflow/providers/imap/sensors/imap_attachment.py +++ b/airflow/providers/imap/sensors/imap_attachment.py @@ -18,7 +18,6 @@ """This module allows you to poke for attachments on a mail server.""" from airflow.providers.imap.hooks.imap import ImapHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class ImapAttachmentSensor(BaseSensorOperator): @@ -42,7 +41,6 @@ class ImapAttachmentSensor(BaseSensorOperator): template_fields = ('attachment_name', 'mail_filter') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/jdbc/operators/jdbc.py b/airflow/providers/jdbc/operators/jdbc.py index 8c0476f0dacb6..3676cd0b3874c 100644 --- a/airflow/providers/jdbc/operators/jdbc.py +++ b/airflow/providers/jdbc/operators/jdbc.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.jdbc.hooks.jdbc import JdbcHook -from airflow.utils.decorators import apply_defaults class JdbcOperator(BaseOperator): @@ -49,7 +48,6 @@ class JdbcOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#ededed' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/jenkins/operators/jenkins_job_trigger.py b/airflow/providers/jenkins/operators/jenkins_job_trigger.py index 8284f13c6f6f5..7044486ee4452 100644 --- a/airflow/providers/jenkins/operators/jenkins_job_trigger.py +++ b/airflow/providers/jenkins/operators/jenkins_job_trigger.py @@ -30,7 +30,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.jenkins.hooks.jenkins import JenkinsHook -from airflow.utils.decorators import apply_defaults JenkinsRequest = Mapping[str, Any] ParamType = Optional[Union[str, Dict, List]] @@ -100,7 +99,6 @@ class JenkinsJobTriggerOperator(BaseOperator): template_ext = ('.json',) ui_color = '#f9ec86' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/jira/operators/jira.py b/airflow/providers/jira/operators/jira.py index 1e9530c85136a..1773f64627d2d 100644 --- a/airflow/providers/jira/operators/jira.py +++ b/airflow/providers/jira/operators/jira.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.jira.hooks.jira import JIRAError, JiraHook -from airflow.utils.decorators import apply_defaults class JiraOperator(BaseOperator): @@ -44,7 +43,6 @@ class JiraOperator(BaseOperator): template_fields = ("jira_method_args",) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/jira/sensors/jira.py b/airflow/providers/jira/sensors/jira.py index 188e98e26065e..0d9321206da37 100644 --- a/airflow/providers/jira/sensors/jira.py +++ b/airflow/providers/jira/sensors/jira.py @@ -21,7 +21,6 @@ from airflow.providers.jira.operators.jira import JIRAError, JiraOperator from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class JiraSensor(BaseSensorOperator): @@ -38,7 +37,6 @@ class JiraSensor(BaseSensorOperator): :type result_processor: function """ - @apply_defaults def __init__( self, *, @@ -85,7 +83,6 @@ class JiraTicketSensor(JiraSensor): template_fields = ("ticket_id",) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/example_dags/example_azure_blob_to_gcs.py b/airflow/providers/microsoft/azure/example_dags/example_azure_blob_to_gcs.py index 693286fe00858..58e582b1f7a32 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_azure_blob_to_gcs.py +++ b/airflow/providers/microsoft/azure/example_dags/example_azure_blob_to_gcs.py @@ -25,7 +25,7 @@ BLOB_NAME = os.environ.get("AZURE_BLOB_NAME", "file.txt") AZURE_CONTAINER_NAME = os.environ.get("AZURE_CONTAINER_NAME", "airflow") GCP_BUCKET_FILE_PATH = os.environ.get("GCP_BUCKET_FILE_PATH", "file.txt") -GCP_BUCKET_NAME = os.environ.get("GCP_BUCKET_NAME", "azure_bucket") +GCP_BUCKET_NAME = os.environ.get("GCP_BUCKET_NAME", "INVALID BUCKET NAME") GCP_OBJECT_NAME = os.environ.get("GCP_OBJECT_NAME", "file.txt") diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index 868a2c7fcaae4..eccfe1ae5aa61 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -389,6 +389,7 @@ def delete_file( blob_name: str, is_prefix: bool = False, ignore_if_missing: bool = False, + delimiter: str = '', **kwargs, ) -> None: """ @@ -407,7 +408,9 @@ def delete_file( :type kwargs: object """ if is_prefix: - blobs_to_delete = self.get_blobs_list(container_name, prefix=blob_name, **kwargs) + blobs_to_delete = self.get_blobs_list( + container_name, prefix=blob_name, delimiter=delimiter, **kwargs + ) elif self.check_for_blob(container_name, blob_name): blobs_to_delete = [blob_name] else: diff --git a/airflow/providers/microsoft/azure/operators/adls_delete.py b/airflow/providers/microsoft/azure/operators/adls_delete.py index 93743334f81ef..643bea27024ec 100644 --- a/airflow/providers/microsoft/azure/operators/adls_delete.py +++ b/airflow/providers/microsoft/azure/operators/adls_delete.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.microsoft.azure.hooks.azure_data_lake import AzureDataLakeHook -from airflow.utils.decorators import apply_defaults class AzureDataLakeStorageDeleteOperator(BaseOperator): @@ -43,7 +42,6 @@ class AzureDataLakeStorageDeleteOperator(BaseOperator): template_fields: Sequence[str] = ('path',) ui_color = '#901dd2' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/operators/adls_list.py b/airflow/providers/microsoft/azure/operators/adls_list.py index a9a2296339624..ba2586509612e 100644 --- a/airflow/providers/microsoft/azure/operators/adls_list.py +++ b/airflow/providers/microsoft/azure/operators/adls_list.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.microsoft.azure.hooks.azure_data_lake import AzureDataLakeHook -from airflow.utils.decorators import apply_defaults class AzureDataLakeStorageListOperator(BaseOperator): @@ -49,7 +48,6 @@ class AzureDataLakeStorageListOperator(BaseOperator): template_fields: Sequence[str] = ('path',) ui_color = '#901dd2' - @apply_defaults def __init__( self, *, path: str, azure_data_lake_conn_id: str = 'azure_data_lake_default', **kwargs ) -> None: diff --git a/airflow/providers/microsoft/azure/operators/adx.py b/airflow/providers/microsoft/azure/operators/adx.py index d8ac1f931eb5f..23af6effd0a5f 100644 --- a/airflow/providers/microsoft/azure/operators/adx.py +++ b/airflow/providers/microsoft/azure/operators/adx.py @@ -25,7 +25,6 @@ from airflow.configuration import conf from airflow.models import BaseOperator from airflow.providers.microsoft.azure.hooks.adx import AzureDataExplorerHook -from airflow.utils.decorators import apply_defaults class AzureDataExplorerQueryOperator(BaseOperator): @@ -48,7 +47,6 @@ class AzureDataExplorerQueryOperator(BaseOperator): template_fields = ('query', 'database') template_ext = ('.kql',) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/operators/azure_batch.py b/airflow/providers/microsoft/azure/operators/azure_batch.py index b8737807c1641..279c3b119a872 100644 --- a/airflow/providers/microsoft/azure/operators/azure_batch.py +++ b/airflow/providers/microsoft/azure/operators/azure_batch.py @@ -23,7 +23,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.microsoft.azure.hooks.azure_batch import AzureBatchHook -from airflow.utils.decorators import apply_defaults # pylint: disable=too-many-instance-attributes @@ -132,7 +131,6 @@ class AzureBatchOperator(BaseOperator): ) ui_color = '#f0f0e4' - @apply_defaults def __init__( self, *, # pylint: disable=too-many-arguments,too-many-locals diff --git a/airflow/providers/microsoft/azure/operators/azure_container_instances.py b/airflow/providers/microsoft/azure/operators/azure_container_instances.py index a4ddf231c8bf8..bbe660f267799 100644 --- a/airflow/providers/microsoft/azure/operators/azure_container_instances.py +++ b/airflow/providers/microsoft/azure/operators/azure_container_instances.py @@ -38,7 +38,6 @@ from airflow.providers.microsoft.azure.hooks.azure_container_instance import AzureContainerInstanceHook from airflow.providers.microsoft.azure.hooks.azure_container_registry import AzureContainerRegistryHook from airflow.providers.microsoft.azure.hooks.azure_container_volume import AzureContainerVolumeHook -from airflow.utils.decorators import apply_defaults Volume = namedtuple( 'Volume', @@ -137,7 +136,6 @@ class AzureContainerInstancesOperator(BaseOperator): template_fields_renderers = {"command": "bash", "environment_variables": "json"} # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/operators/azure_cosmos.py b/airflow/providers/microsoft/azure/operators/azure_cosmos.py index c4cc0eee5e880..b096bcc8c8af4 100644 --- a/airflow/providers/microsoft/azure/operators/azure_cosmos.py +++ b/airflow/providers/microsoft/azure/operators/azure_cosmos.py @@ -18,7 +18,6 @@ from airflow.models import BaseOperator from airflow.providers.microsoft.azure.hooks.azure_cosmos import AzureCosmosDBHook -from airflow.utils.decorators import apply_defaults class AzureCosmosInsertDocumentOperator(BaseOperator): @@ -40,7 +39,6 @@ class AzureCosmosInsertDocumentOperator(BaseOperator): template_fields = ('database_name', 'collection_name') ui_color = '#e4f0e8' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/operators/wasb_delete_blob.py b/airflow/providers/microsoft/azure/operators/wasb_delete_blob.py index 58c41b907dd9a..3e8c1c5726a4c 100644 --- a/airflow/providers/microsoft/azure/operators/wasb_delete_blob.py +++ b/airflow/providers/microsoft/azure/operators/wasb_delete_blob.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.microsoft.azure.hooks.wasb import WasbHook -from airflow.utils.decorators import apply_defaults class WasbDeleteBlobOperator(BaseOperator): @@ -44,7 +43,6 @@ class WasbDeleteBlobOperator(BaseOperator): template_fields = ('container_name', 'blob_name') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/sensors/azure_cosmos.py b/airflow/providers/microsoft/azure/sensors/azure_cosmos.py index 1f1a48c799857..277e193fff6af 100644 --- a/airflow/providers/microsoft/azure/sensors/azure_cosmos.py +++ b/airflow/providers/microsoft/azure/sensors/azure_cosmos.py @@ -18,7 +18,6 @@ from airflow.providers.microsoft.azure.hooks.azure_cosmos import AzureCosmosDBHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class AzureCosmosDocumentSensor(BaseSensorOperator): @@ -48,7 +47,6 @@ class AzureCosmosDocumentSensor(BaseSensorOperator): template_fields = ('database_name', 'collection_name', 'document_id') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/sensors/wasb.py b/airflow/providers/microsoft/azure/sensors/wasb.py index a4adec6778c38..8a345bf452b36 100644 --- a/airflow/providers/microsoft/azure/sensors/wasb.py +++ b/airflow/providers/microsoft/azure/sensors/wasb.py @@ -20,7 +20,6 @@ from airflow.providers.microsoft.azure.hooks.wasb import WasbHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class WasbBlobSensor(BaseSensorOperator): @@ -40,7 +39,6 @@ class WasbBlobSensor(BaseSensorOperator): template_fields = ('container_name', 'blob_name') - @apply_defaults def __init__( self, *, @@ -81,7 +79,6 @@ class WasbPrefixSensor(BaseSensorOperator): template_fields = ('container_name', 'prefix') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py b/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py index ccc7577a38f89..1da2e084d139d 100644 --- a/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py +++ b/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py @@ -22,7 +22,6 @@ from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.microsoft.azure.hooks.wasb import WasbHook -from airflow.utils.decorators import apply_defaults class AzureBlobStorageToGCSOperator(BaseOperator): @@ -66,7 +65,6 @@ class AzureBlobStorageToGCSOperator(BaseOperator): :type impersonation_chain: Union[str, Sequence[str]] """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/transfers/file_to_wasb.py b/airflow/providers/microsoft/azure/transfers/file_to_wasb.py index c099faa23c271..0d00ded974d1f 100644 --- a/airflow/providers/microsoft/azure/transfers/file_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/file_to_wasb.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.microsoft.azure.hooks.wasb import WasbHook -from airflow.utils.decorators import apply_defaults class FileToWasbOperator(BaseOperator): @@ -42,7 +41,6 @@ class FileToWasbOperator(BaseOperator): template_fields = ('file_path', 'container_name', 'blob_name') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/transfers/local_to_adls.py b/airflow/providers/microsoft/azure/transfers/local_to_adls.py index bf6947b653de7..aaf8bbf79506f 100644 --- a/airflow/providers/microsoft/azure/transfers/local_to_adls.py +++ b/airflow/providers/microsoft/azure/transfers/local_to_adls.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.microsoft.azure.hooks.azure_data_lake import AzureDataLakeHook -from airflow.utils.decorators import apply_defaults class LocalToAzureDataLakeStorageOperator(BaseOperator): @@ -62,7 +61,6 @@ class LocalToAzureDataLakeStorageOperator(BaseOperator): template_fields = ("local_path", "remote_path") ui_color = '#e4f0e8' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py b/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py index 3ce1e2412ec7c..347473f2550fd 100644 --- a/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py +++ b/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py @@ -25,7 +25,6 @@ from airflow.models import BaseOperator from airflow.providers.microsoft.azure.hooks.azure_data_lake import AzureDataLakeHook from airflow.providers.oracle.hooks.oracle import OracleHook -from airflow.utils.decorators import apply_defaults class OracleToAzureDataLakeOperator(BaseOperator): @@ -40,7 +39,7 @@ class OracleToAzureDataLakeOperator(BaseOperator): :type azure_data_lake_conn_id: str :param azure_data_lake_path: destination path in azure data lake to put the file. :type azure_data_lake_path: str - :param oracle_conn_id: source Oracle connection. + :param oracle_conn_id: :ref:`Source Oracle connection `. :type oracle_conn_id: str :param sql: SQL query to execute against the Oracle database. (templated) :type sql: str @@ -61,7 +60,6 @@ class OracleToAzureDataLakeOperator(BaseOperator): ui_color = '#e08c8c' # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/mssql/operators/mssql.py b/airflow/providers/microsoft/mssql/operators/mssql.py index 72fda3239a17b..bcd8a2d4a5370 100644 --- a/airflow/providers/microsoft/mssql/operators/mssql.py +++ b/airflow/providers/microsoft/mssql/operators/mssql.py @@ -20,7 +20,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.microsoft.mssql.hooks.mssql import MsSqlHook -from airflow.utils.decorators import apply_defaults if TYPE_CHECKING: from airflow.hooks.dbapi import DbApiHook @@ -53,7 +52,6 @@ class MsSqlOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#ededed' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/microsoft/winrm/operators/winrm.py b/airflow/providers/microsoft/winrm/operators/winrm.py index 8a2952957be7f..816df703876e0 100644 --- a/airflow/providers/microsoft/winrm/operators/winrm.py +++ b/airflow/providers/microsoft/winrm/operators/winrm.py @@ -26,7 +26,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.microsoft.winrm.hooks.winrm import WinRMHook -from airflow.utils.decorators import apply_defaults # Hide the following error message in urllib3 when making WinRM connections: # requests.packages.urllib3.exceptions.HeaderParsingError: [StartBoundaryNotFoundDefect(), @@ -58,7 +57,6 @@ class WinRMOperator(BaseOperator): template_fields = ('command',) template_fields_renderers = {"command": "powershell"} - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/mongo/sensors/mongo.py b/airflow/providers/mongo/sensors/mongo.py index e8d67fea543c3..cdd48c76986d6 100644 --- a/airflow/providers/mongo/sensors/mongo.py +++ b/airflow/providers/mongo/sensors/mongo.py @@ -17,7 +17,6 @@ # under the License. from airflow.providers.mongo.hooks.mongo import MongoHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class MongoSensor(BaseSensorOperator): @@ -41,7 +40,6 @@ class MongoSensor(BaseSensorOperator): template_fields = ('collection', 'query') - @apply_defaults def __init__( self, *, collection: str, query: dict, mongo_conn_id: str = "mongo_default", **kwargs ) -> None: diff --git a/airflow/providers/mysql/operators/mysql.py b/airflow/providers/mysql/operators/mysql.py index 8d0797ad31846..d8cf15d3f2088 100644 --- a/airflow/providers/mysql/operators/mysql.py +++ b/airflow/providers/mysql/operators/mysql.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.mysql.hooks.mysql import MySqlHook -from airflow.utils.decorators import apply_defaults class MySqlOperator(BaseOperator): @@ -50,7 +49,6 @@ class MySqlOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#ededed' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/mysql/transfers/presto_to_mysql.py b/airflow/providers/mysql/transfers/presto_to_mysql.py index 2c59f7d7185dc..7d776507167f7 100644 --- a/airflow/providers/mysql/transfers/presto_to_mysql.py +++ b/airflow/providers/mysql/transfers/presto_to_mysql.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.mysql.hooks.mysql import MySqlHook from airflow.providers.presto.hooks.presto import PrestoHook -from airflow.utils.decorators import apply_defaults class PrestoToMySqlOperator(BaseOperator): @@ -50,7 +49,6 @@ class PrestoToMySqlOperator(BaseOperator): template_fields_renderers = {"mysql_preoperator": "sql"} ui_color = '#a0e08c' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/mysql/transfers/s3_to_mysql.py b/airflow/providers/mysql/transfers/s3_to_mysql.py index 91ca7438ab2bc..fcc8ae4da6626 100644 --- a/airflow/providers/mysql/transfers/s3_to_mysql.py +++ b/airflow/providers/mysql/transfers/s3_to_mysql.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.mysql.hooks.mysql import MySqlHook -from airflow.utils.decorators import apply_defaults class S3ToMySqlOperator(BaseOperator): @@ -53,7 +52,6 @@ class S3ToMySqlOperator(BaseOperator): template_ext = () ui_color = '#f4a460' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/mysql/transfers/trino_to_mysql.py b/airflow/providers/mysql/transfers/trino_to_mysql.py index e8091497551e9..ba10b1f5542a1 100644 --- a/airflow/providers/mysql/transfers/trino_to_mysql.py +++ b/airflow/providers/mysql/transfers/trino_to_mysql.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.mysql.hooks.mysql import MySqlHook from airflow.providers.trino.hooks.trino import TrinoHook -from airflow.utils.decorators import apply_defaults class TrinoToMySqlOperator(BaseOperator): @@ -50,7 +49,6 @@ class TrinoToMySqlOperator(BaseOperator): template_fields_renderers = {"mysql_preoperator": "sql"} ui_color = '#a0e08c' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/mysql/transfers/vertica_to_mysql.py b/airflow/providers/mysql/transfers/vertica_to_mysql.py index 8f5b1dbbfa463..03747a0fd4873 100644 --- a/airflow/providers/mysql/transfers/vertica_to_mysql.py +++ b/airflow/providers/mysql/transfers/vertica_to_mysql.py @@ -26,7 +26,6 @@ from airflow.models import BaseOperator from airflow.providers.mysql.hooks.mysql import MySqlHook from airflow.providers.vertica.hooks.vertica import VerticaHook -from airflow.utils.decorators import apply_defaults class VerticaToMySqlOperator(BaseOperator): @@ -66,7 +65,6 @@ class VerticaToMySqlOperator(BaseOperator): } ui_color = '#a0e08c' - @apply_defaults def __init__( self, sql: str, diff --git a/airflow/providers/neo4j/operators/neo4j.py b/airflow/providers/neo4j/operators/neo4j.py index 6cff3f84b678f..5a14e46934cff 100644 --- a/airflow/providers/neo4j/operators/neo4j.py +++ b/airflow/providers/neo4j/operators/neo4j.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.neo4j.hooks.neo4j import Neo4jHook -from airflow.utils.decorators import apply_defaults class Neo4jOperator(BaseOperator): @@ -37,7 +36,6 @@ class Neo4jOperator(BaseOperator): :type neo4j_conn_id: str """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/opsgenie/operators/opsgenie_alert.py b/airflow/providers/opsgenie/operators/opsgenie_alert.py index 5a16a08bcdfe9..1cdca1c67604b 100644 --- a/airflow/providers/opsgenie/operators/opsgenie_alert.py +++ b/airflow/providers/opsgenie/operators/opsgenie_alert.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.opsgenie.hooks.opsgenie_alert import OpsgenieAlertHook -from airflow.utils.decorators import apply_defaults class OpsgenieAlertOperator(BaseOperator): @@ -70,7 +69,6 @@ class OpsgenieAlertOperator(BaseOperator): template_fields = ('message', 'alias', 'description', 'entity', 'priority', 'note') # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/oracle/hooks/oracle.py b/airflow/providers/oracle/hooks/oracle.py index 73004bdec1084..d11f77d9e50d6 100644 --- a/airflow/providers/oracle/hooks/oracle.py +++ b/airflow/providers/oracle/hooks/oracle.py @@ -29,7 +29,8 @@ class OracleHook(DbApiHook): """ Interact with Oracle SQL. - :param oracle_conn_id: The Airflow connection used for Oracle credentials. + :param oracle_conn_id: The :ref:`Oracle connection id ` + used for Oracle credentials. :type oracle_conn_id: str """ @@ -50,12 +51,24 @@ def get_conn(self) -> 'OracleHook': (from the Oracle names server or tnsnames.ora file) or is a string like the one returned from makedsn(). - :param dsn: the host address for the Oracle server + :param dsn: the data source name for the Oracle server :param service_name: the db_unique_name of the database that you are connecting to (CONNECT_DATA part of TNS) + :param sid: Oracle System ID that identifies a particular + database on a system You can set these parameters in the extra fields of your connection - as in ``{ "dsn":"some.host.address" , "service_name":"some.service.name" }`` + as in + + .. code-block:: python + + { + "dsn": ( + "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)" + "(HOST=host)(PORT=1521))(CONNECT_DATA=(SID=sid)))" + ) + } + see more param detail in `cx_Oracle.connect `_ @@ -65,18 +78,24 @@ def get_conn(self) -> 'OracleHook': self.oracle_conn_id # type: ignore[attr-defined] # pylint: disable=no-member ) conn_config = {'user': conn.login, 'password': conn.password} - dsn = conn.extra_dejson.get('dsn') sid = conn.extra_dejson.get('sid') mod = conn.extra_dejson.get('module') service_name = conn.extra_dejson.get('service_name') port = conn.port if conn.port else 1521 - if dsn and sid and not service_name: - conn_config['dsn'] = cx_Oracle.makedsn(dsn, port, sid) - elif dsn and service_name and not sid: - conn_config['dsn'] = cx_Oracle.makedsn(dsn, port, service_name=service_name) + if conn.host and sid and not service_name: + conn_config['dsn'] = cx_Oracle.makedsn(conn.host, port, sid) + elif conn.host and service_name and not sid: + conn_config['dsn'] = cx_Oracle.makedsn(conn.host, port, service_name=service_name) else: - conn_config['dsn'] = conn.host + dsn = conn.extra_dejson.get('dsn') + if dsn is None: + dsn = conn.host + if conn.port is not None: + dsn += ":" + str(conn.port) + if service_name or conn.schema: + dsn += "/" + (service_name or conn.schema) + conn_config['dsn'] = dsn if 'encoding' in conn.extra_dejson: conn_config['encoding'] = conn.extra_dejson.get('encoding') diff --git a/airflow/providers/oracle/operators/oracle.py b/airflow/providers/oracle/operators/oracle.py index 2cd3cbc47d943..d764225855f3d 100644 --- a/airflow/providers/oracle/operators/oracle.py +++ b/airflow/providers/oracle/operators/oracle.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.oracle.hooks.oracle import OracleHook -from airflow.utils.decorators import apply_defaults class OracleOperator(BaseOperator): @@ -31,7 +30,8 @@ class OracleOperator(BaseOperator): Template reference are recognized by str ending in '.sql' (templated) :type sql: str or list[str] - :param oracle_conn_id: reference to a specific Oracle database + :param oracle_conn_id: The :ref:`Oracle connection id ` + reference to a specific Oracle database. :type oracle_conn_id: str :param parameters: (optional) the parameters to render the SQL query with. :type parameters: dict or iterable @@ -44,7 +44,6 @@ class OracleOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#ededed' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/oracle/transfers/oracle_to_oracle.py b/airflow/providers/oracle/transfers/oracle_to_oracle.py index be09874d70d5e..fd4b939ac6ef4 100644 --- a/airflow/providers/oracle/transfers/oracle_to_oracle.py +++ b/airflow/providers/oracle/transfers/oracle_to_oracle.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.oracle.hooks.oracle import OracleHook -from airflow.utils.decorators import apply_defaults class OracleToOracleOperator(BaseOperator): @@ -31,7 +30,7 @@ class OracleToOracleOperator(BaseOperator): :type oracle_destination_conn_id: str :param destination_table: destination table to insert rows. :type destination_table: str - :param oracle_source_conn_id: source Oracle connection. + :param oracle_source_conn_id: :ref:`Source Oracle connection `. :type oracle_source_conn_id: str :param source_sql: SQL query to execute against the source Oracle database. (templated) @@ -46,7 +45,6 @@ class OracleToOracleOperator(BaseOperator): template_fields_renderers = {"source_sql": "sql", "source_sql_params": "py"} ui_color = '#e08c8c' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/papermill/example_dags/example_papermill.py b/airflow/providers/papermill/example_dags/example_papermill.py index c9e645a331f71..8c6bf2817956f 100644 --- a/airflow/providers/papermill/example_dags/example_papermill.py +++ b/airflow/providers/papermill/example_dags/example_papermill.py @@ -68,7 +68,7 @@ def check_notebook(inlets, execution_date): with DAG( - dag_id='example_papermill_operator', + dag_id='example_papermill_operator_2', default_args=default_args, schedule_interval='0 0 * * *', start_date=days_ago(2), diff --git a/airflow/providers/papermill/operators/papermill.py b/airflow/providers/papermill/operators/papermill.py index cbf528d4c7875..3a13f3a0fdbc7 100644 --- a/airflow/providers/papermill/operators/papermill.py +++ b/airflow/providers/papermill/operators/papermill.py @@ -22,7 +22,6 @@ from airflow.lineage.entities import File from airflow.models import BaseOperator -from airflow.utils.decorators import apply_defaults @attr.s(auto_attribs=True) @@ -49,7 +48,6 @@ class PapermillOperator(BaseOperator): supports_lineage = True - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/plexus/operators/job.py b/airflow/providers/plexus/operators/job.py index 8f56987a0faa3..e20d64a1eac5d 100644 --- a/airflow/providers/plexus/operators/job.py +++ b/airflow/providers/plexus/operators/job.py @@ -24,7 +24,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.plexus.hooks.plexus import PlexusHook -from airflow.utils.decorators import apply_defaults logger = logging.getLogger(__name__) @@ -45,7 +44,6 @@ class PlexusJobOperator(BaseOperator): """ - @apply_defaults def __init__(self, job_params: Dict, **kwargs) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/postgres/operators/postgres.py b/airflow/providers/postgres/operators/postgres.py index f378c82383403..b28cd48678ed1 100644 --- a/airflow/providers/postgres/operators/postgres.py +++ b/airflow/providers/postgres/operators/postgres.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.postgres.hooks.postgres import PostgresHook -from airflow.utils.decorators import apply_defaults class PostgresOperator(BaseOperator): @@ -47,7 +46,6 @@ class PostgresOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#ededed' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 0610662c72c33..ff0ba9e6d3c0f 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -31,7 +31,7 @@ integrations: external-doc-url: https://www.postgresql.org/ how-to-guide: - /docs/apache-airflow-providers-postgres/operators/postgres_operator_howto_guide.rst - logo: /integration-logos/postgress/Postgress.png + logo: /integration-logos/postgres/Postgres.png tags: [software] operators: diff --git a/airflow/providers/qubole/operators/qubole.py b/airflow/providers/qubole/operators/qubole.py index ce8a6c7c1809d..f58f2b9039f94 100644 --- a/airflow/providers/qubole/operators/qubole.py +++ b/airflow/providers/qubole/operators/qubole.py @@ -30,7 +30,6 @@ QuboleHook, flatten_list, ) -from airflow.utils.decorators import apply_defaults class QDSLink(BaseOperatorLink): @@ -218,7 +217,6 @@ class QuboleOperator(BaseOperator): operator_extra_links = (QDSLink(),) - @apply_defaults def __init__(self, *, qubole_conn_id: str = "qubole_default", **kwargs) -> None: self.kwargs = kwargs self.kwargs['qubole_conn_id'] = qubole_conn_id diff --git a/airflow/providers/qubole/operators/qubole_check.py b/airflow/providers/qubole/operators/qubole_check.py index 45534e0235c17..6f9f9250e165e 100644 --- a/airflow/providers/qubole/operators/qubole_check.py +++ b/airflow/providers/qubole/operators/qubole_check.py @@ -22,7 +22,6 @@ from airflow.operators.sql import SQLCheckOperator, SQLValueCheckOperator from airflow.providers.qubole.hooks.qubole_check import QuboleCheckHook from airflow.providers.qubole.operators.qubole import QuboleOperator -from airflow.utils.decorators import apply_defaults class _QuboleCheckOperatorMixin: @@ -104,7 +103,6 @@ class QuboleCheckOperator(_QuboleCheckOperatorMixin, SQLCheckOperator, QuboleOpe template_ext = QuboleOperator.template_ext ui_fgcolor = '#000' - @apply_defaults def __init__( self, *, qubole_conn_id: str = "qubole_default", results_parser_callable: Callable = None, **kwargs ) -> None: @@ -161,7 +159,6 @@ class QuboleValueCheckOperator(_QuboleCheckOperatorMixin, SQLValueCheckOperator, template_ext = QuboleOperator.template_ext ui_fgcolor = '#000' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/qubole/sensors/qubole.py b/airflow/providers/qubole/sensors/qubole.py index 428e03f962690..044eb672d1f8a 100644 --- a/airflow/providers/qubole/sensors/qubole.py +++ b/airflow/providers/qubole/sensors/qubole.py @@ -22,7 +22,6 @@ from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class QuboleSensor(BaseSensorOperator): @@ -32,7 +31,6 @@ class QuboleSensor(BaseSensorOperator): template_ext = ('.txt',) - @apply_defaults def __init__(self, *, data, qubole_conn_id: str = "qubole_default", **kwargs) -> None: self.data = data self.qubole_conn_id = qubole_conn_id @@ -83,7 +81,6 @@ class QuboleFileSensor(QuboleSensor): also use ``.txt`` files for template-driven use cases. """ - @apply_defaults def __init__(self, **kwargs) -> None: self.sensor_class = FileSensor super().__init__(**kwargs) @@ -106,7 +103,6 @@ class QubolePartitionSensor(QuboleSensor): also use ``.txt`` files for template-driven use cases. """ - @apply_defaults def __init__(self, **kwargs) -> None: self.sensor_class = PartitionSensor super().__init__(**kwargs) diff --git a/airflow/providers/redis/operators/redis_publish.py b/airflow/providers/redis/operators/redis_publish.py index 97f2c364e82a8..c3858ef14fcff 100644 --- a/airflow/providers/redis/operators/redis_publish.py +++ b/airflow/providers/redis/operators/redis_publish.py @@ -20,7 +20,6 @@ from airflow.models import BaseOperator from airflow.providers.redis.hooks.redis import RedisHook -from airflow.utils.decorators import apply_defaults class RedisPublishOperator(BaseOperator): @@ -37,7 +36,6 @@ class RedisPublishOperator(BaseOperator): template_fields = ('channel', 'message') - @apply_defaults def __init__(self, *, channel: str, message: str, redis_conn_id: str = 'redis_default', **kwargs) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/redis/sensors/redis_key.py b/airflow/providers/redis/sensors/redis_key.py index b32888560789c..265ff6ceae585 100644 --- a/airflow/providers/redis/sensors/redis_key.py +++ b/airflow/providers/redis/sensors/redis_key.py @@ -19,7 +19,6 @@ from airflow.providers.redis.hooks.redis import RedisHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class RedisKeySensor(BaseSensorOperator): @@ -28,7 +27,6 @@ class RedisKeySensor(BaseSensorOperator): template_fields = ('key',) ui_color = '#f0eee4' - @apply_defaults def __init__(self, *, key: str, redis_conn_id: str, **kwargs) -> None: super().__init__(**kwargs) self.redis_conn_id = redis_conn_id diff --git a/airflow/providers/redis/sensors/redis_pub_sub.py b/airflow/providers/redis/sensors/redis_pub_sub.py index 28cab6b16b2c9..c618072938569 100644 --- a/airflow/providers/redis/sensors/redis_pub_sub.py +++ b/airflow/providers/redis/sensors/redis_pub_sub.py @@ -20,7 +20,6 @@ from airflow.providers.redis.hooks.redis import RedisHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class RedisPubSubSensor(BaseSensorOperator): @@ -36,7 +35,6 @@ class RedisPubSubSensor(BaseSensorOperator): template_fields = ('channels',) ui_color = '#f0eee4' - @apply_defaults def __init__(self, *, channels: Union[List[str], str], redis_conn_id: str, **kwargs) -> None: super().__init__(**kwargs) self.channels = channels diff --git a/airflow/providers/segment/operators/segment_track_event.py b/airflow/providers/segment/operators/segment_track_event.py index eda6859e5e079..4a5f122482b1e 100644 --- a/airflow/providers/segment/operators/segment_track_event.py +++ b/airflow/providers/segment/operators/segment_track_event.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.segment.hooks.segment import SegmentHook -from airflow.utils.decorators import apply_defaults class SegmentTrackEventOperator(BaseOperator): @@ -42,7 +41,6 @@ class SegmentTrackEventOperator(BaseOperator): template_fields = ('user_id', 'event', 'properties') ui_color = '#ffd700' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/sftp/operators/sftp.py b/airflow/providers/sftp/operators/sftp.py index 63b9ab0ebf8f8..cb2f24f759ca0 100644 --- a/airflow/providers/sftp/operators/sftp.py +++ b/airflow/providers/sftp/operators/sftp.py @@ -23,7 +23,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.ssh.hooks.ssh import SSHHook -from airflow.utils.decorators import apply_defaults class SFTPOperation: @@ -81,7 +80,6 @@ class SFTPOperator(BaseOperator): template_fields = ('local_filepath', 'remote_filepath', 'remote_host') - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/sftp/sensors/sftp.py b/airflow/providers/sftp/sensors/sftp.py index 564fc91a72f92..72c781cb0dc92 100644 --- a/airflow/providers/sftp/sensors/sftp.py +++ b/airflow/providers/sftp/sensors/sftp.py @@ -22,7 +22,6 @@ from airflow.providers.sftp.hooks.sftp import SFTPHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class SFTPSensor(BaseSensorOperator): @@ -37,7 +36,6 @@ class SFTPSensor(BaseSensorOperator): template_fields = ('path',) - @apply_defaults def __init__(self, *, path: str, sftp_conn_id: str = 'sftp_default', **kwargs) -> None: super().__init__(**kwargs) self.path = path diff --git a/airflow/providers/singularity/operators/singularity.py b/airflow/providers/singularity/operators/singularity.py index 0423a25d2497d..a3c8c9676f3fe 100644 --- a/airflow/providers/singularity/operators/singularity.py +++ b/airflow/providers/singularity/operators/singularity.py @@ -25,7 +25,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator -from airflow.utils.decorators import apply_defaults class SingularityOperator(BaseOperator): @@ -72,7 +71,6 @@ class SingularityOperator(BaseOperator): ) template_fields_renderers = {"command": "bash", "environment": "json"} - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/slack/operators/slack.py b/airflow/providers/slack/operators/slack.py index 9615bc5edea24..1b88a801ff760 100644 --- a/airflow/providers/slack/operators/slack.py +++ b/airflow/providers/slack/operators/slack.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.slack.hooks.slack import SlackHook -from airflow.utils.decorators import apply_defaults class SlackAPIOperator(BaseOperator): @@ -44,7 +43,6 @@ class SlackAPIOperator(BaseOperator): :type client_args: dict """ - @apply_defaults def __init__( self, *, @@ -123,7 +121,6 @@ class SlackAPIPostOperator(SlackAPIOperator): template_fields = ('username', 'text', 'attachments', 'blocks', 'channel') ui_color = '#FFBA40' - @apply_defaults def __init__( self, channel: str = '#general', @@ -191,7 +188,6 @@ class SlackAPIFileOperator(SlackAPIOperator): template_fields = ('channel', 'initial_comment', 'filename', 'filetype', 'content') ui_color = '#44BEDF' - @apply_defaults def __init__( self, channel: str = '#general', diff --git a/airflow/providers/slack/operators/slack_webhook.py b/airflow/providers/slack/operators/slack_webhook.py index 1a7725a81c6ae..098324e2448c5 100644 --- a/airflow/providers/slack/operators/slack_webhook.py +++ b/airflow/providers/slack/operators/slack_webhook.py @@ -20,7 +20,6 @@ from airflow.providers.http.operators.http import SimpleHttpOperator from airflow.providers.slack.hooks.slack_webhook import SlackWebhookHook -from airflow.utils.decorators import apply_defaults class SlackWebhookOperator(SimpleHttpOperator): @@ -71,7 +70,6 @@ class SlackWebhookOperator(SimpleHttpOperator): ] # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index d2ca43b3da207..70a92053c884e 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook -from airflow.utils.decorators import apply_defaults class SnowflakeOperator(BaseOperator): @@ -71,7 +70,6 @@ class SnowflakeOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#ededed' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/snowflake/transfers/s3_to_snowflake.py b/airflow/providers/snowflake/transfers/s3_to_snowflake.py index 1e4e8772eea01..cebf54695b8ac 100644 --- a/airflow/providers/snowflake/transfers/s3_to_snowflake.py +++ b/airflow/providers/snowflake/transfers/s3_to_snowflake.py @@ -21,7 +21,6 @@ from airflow.models import BaseOperator from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook -from airflow.utils.decorators import apply_defaults class S3ToSnowflakeOperator(BaseOperator): @@ -72,7 +71,6 @@ class S3ToSnowflakeOperator(BaseOperator): :type session_parameters: dict """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/snowflake/transfers/snowflake_to_slack.py b/airflow/providers/snowflake/transfers/snowflake_to_slack.py index a07e85fc4310f..a13841c058bae 100644 --- a/airflow/providers/snowflake/transfers/snowflake_to_slack.py +++ b/airflow/providers/snowflake/transfers/snowflake_to_slack.py @@ -24,7 +24,6 @@ from airflow.models import BaseOperator from airflow.providers.slack.hooks.slack_webhook import SlackWebhookHook from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook -from airflow.utils.decorators import apply_defaults class SnowflakeToSlackOperator(BaseOperator): @@ -73,7 +72,6 @@ class SnowflakeToSlackOperator(BaseOperator): template_fields_renderers = {"slack_message": "jinja"} times_rendered = 0 - @apply_defaults def __init__( # pylint: disable=too-many-arguments self, *, diff --git a/airflow/providers/sqlite/operators/sqlite.py b/airflow/providers/sqlite/operators/sqlite.py index 74b80ea55fff4..825916c4f212f 100644 --- a/airflow/providers/sqlite/operators/sqlite.py +++ b/airflow/providers/sqlite/operators/sqlite.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.sqlite.hooks.sqlite import SqliteHook -from airflow.utils.decorators import apply_defaults class SqliteOperator(BaseOperator): @@ -45,7 +44,6 @@ class SqliteOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#cdaaed' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/ssh/operators/ssh.py b/airflow/providers/ssh/operators/ssh.py index 5bd7868bc11e7..c82e0e12c30a4 100644 --- a/airflow/providers/ssh/operators/ssh.py +++ b/airflow/providers/ssh/operators/ssh.py @@ -24,7 +24,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.ssh.hooks.ssh import SSHHook -from airflow.utils.decorators import apply_defaults class SSHOperator(BaseOperator): @@ -60,7 +59,6 @@ class SSHOperator(BaseOperator): template_ext = ('.sh',) template_fields_renderers = {"command": "bash"} - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/tableau/operators/tableau_refresh_workbook.py b/airflow/providers/tableau/operators/tableau_refresh_workbook.py index 1015650471d7f..cca01e7b451d0 100644 --- a/airflow/providers/tableau/operators/tableau_refresh_workbook.py +++ b/airflow/providers/tableau/operators/tableau_refresh_workbook.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.tableau.hooks.tableau import TableauHook -from airflow.utils.decorators import apply_defaults class TableauRefreshWorkbookOperator(BaseOperator): @@ -41,7 +40,6 @@ class TableauRefreshWorkbookOperator(BaseOperator): :type tableau_conn_id: str """ - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/tableau/sensors/tableau_job_status.py b/airflow/providers/tableau/sensors/tableau_job_status.py index 87c52e6834396..83c76a28b6c9a 100644 --- a/airflow/providers/tableau/sensors/tableau_job_status.py +++ b/airflow/providers/tableau/sensors/tableau_job_status.py @@ -19,7 +19,6 @@ from airflow.exceptions import AirflowException from airflow.providers.tableau.hooks.tableau import TableauHook, TableauJobFinishCode from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class TableauJobFailedException(AirflowException): @@ -43,7 +42,6 @@ class TableauJobStatusSensor(BaseSensorOperator): template_fields = ('job_id',) - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/telegram/operators/telegram.py b/airflow/providers/telegram/operators/telegram.py index 113a3e4ed56a9..57f6288da063c 100644 --- a/airflow/providers/telegram/operators/telegram.py +++ b/airflow/providers/telegram/operators/telegram.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.telegram.hooks.telegram import TelegramHook -from airflow.utils.decorators import apply_defaults class TelegramOperator(BaseOperator): @@ -49,7 +48,6 @@ class TelegramOperator(BaseOperator): template_fields = ('text', 'chat_id') ui_color = '#FFBA40' - @apply_defaults def __init__( self, *, diff --git a/airflow/providers/vertica/operators/vertica.py b/airflow/providers/vertica/operators/vertica.py index e374365d277bc..c39292621a9d4 100644 --- a/airflow/providers/vertica/operators/vertica.py +++ b/airflow/providers/vertica/operators/vertica.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.vertica.hooks.vertica import VerticaHook -from airflow.utils.decorators import apply_defaults class VerticaOperator(BaseOperator): @@ -38,7 +37,6 @@ class VerticaOperator(BaseOperator): template_ext = ('.sql',) ui_color = '#b4e0ff' - @apply_defaults def __init__( self, *, sql: Union[str, List[str]], vertica_conn_id: str = 'vertica_default', **kwargs: Any ) -> None: diff --git a/airflow/providers/yandex/operators/yandexcloud_dataproc.py b/airflow/providers/yandex/operators/yandexcloud_dataproc.py index 2037c47f6241c..28b2e02270b89 100644 --- a/airflow/providers/yandex/operators/yandexcloud_dataproc.py +++ b/airflow/providers/yandex/operators/yandexcloud_dataproc.py @@ -19,7 +19,6 @@ from airflow.models import BaseOperator from airflow.providers.yandex.hooks.yandexcloud_dataproc import DataprocHook -from airflow.utils.decorators import apply_defaults class DataprocCreateClusterOperator(BaseOperator): @@ -77,7 +76,6 @@ class DataprocCreateClusterOperator(BaseOperator): # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-arguments # pylint: disable=too-many-locals - @apply_defaults def __init__( self, *, @@ -172,7 +170,6 @@ class DataprocDeleteClusterOperator(BaseOperator): template_fields = ['cluster_id'] - @apply_defaults def __init__( self, *, connection_id: Optional[str] = None, cluster_id: Optional[str] = None, **kwargs ) -> None: @@ -217,7 +214,6 @@ class DataprocCreateHiveJobOperator(BaseOperator): template_fields = ['cluster_id'] # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, @@ -291,7 +287,6 @@ class DataprocCreateMapReduceJobOperator(BaseOperator): template_fields = ['cluster_id'] # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, @@ -370,7 +365,6 @@ class DataprocCreateSparkJobOperator(BaseOperator): template_fields = ['cluster_id'] # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, @@ -449,7 +443,6 @@ class DataprocCreatePysparkJobOperator(BaseOperator): template_fields = ['cluster_id'] # pylint: disable=too-many-arguments - @apply_defaults def __init__( self, *, diff --git a/airflow/security/permissions.py b/airflow/security/permissions.py index f4337ebe37140..ae864a078fc96 100644 --- a/airflow/security/permissions.py +++ b/airflow/security/permissions.py @@ -27,6 +27,7 @@ RESOURCE_DOCS = "Documentation" RESOURCE_CONFIG = "Configurations" RESOURCE_CONNECTION = "Connections" +RESOURCE_DAG_DEPENDENCIES = "DAG Dependencies" RESOURCE_DAG_CODE = "DAG Code" RESOURCE_DAG_RUN = "DAG Runs" RESOURCE_IMPORT_ERROR = "ImportError" @@ -38,6 +39,7 @@ RESOURCE_PERMISSION_VIEW = "Permission Views" # Refers to a Perm <-> View mapping, not an MVC View. RESOURCE_POOL = "Pools" RESOURCE_PLUGIN = "Plugins" +RESOURCE_PROVIDER = "Providers" RESOURCE_ROLE = "Roles" RESOURCE_SLA_MISS = "SLA Misses" RESOURCE_TASK_INSTANCE = "Task Instances" diff --git a/airflow/sensors/base.py b/airflow/sensors/base.py index 1349609ff14de..880a4ef4eadea 100644 --- a/airflow/sensors/base.py +++ b/airflow/sensors/base.py @@ -35,7 +35,6 @@ from airflow.models.taskreschedule import TaskReschedule from airflow.ti_deps.deps.ready_to_reschedule import ReadyToRescheduleDep from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults class BaseSensorOperator(BaseOperator, SkipMixin): @@ -89,7 +88,6 @@ class BaseSensorOperator(BaseOperator, SkipMixin): 'email_on_failure', ) - @apply_defaults def __init__( self, *, diff --git a/airflow/sensors/bash.py b/airflow/sensors/bash.py index 21c6ea98c7248..35cd0967770a0 100644 --- a/airflow/sensors/bash.py +++ b/airflow/sensors/bash.py @@ -21,7 +21,6 @@ from tempfile import NamedTemporaryFile, TemporaryDirectory, gettempdir from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class BashSensor(BaseSensorOperator): @@ -44,7 +43,6 @@ class BashSensor(BaseSensorOperator): template_fields = ('bash_command', 'env') - @apply_defaults def __init__(self, *, bash_command, env=None, output_encoding='utf-8', **kwargs): super().__init__(**kwargs) self.bash_command = bash_command diff --git a/airflow/sensors/date_time.py b/airflow/sensors/date_time.py index 4a6f11baaf3e2..3229371a89710 100644 --- a/airflow/sensors/date_time.py +++ b/airflow/sensors/date_time.py @@ -21,7 +21,6 @@ from airflow.sensors.base import BaseSensorOperator from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults class DateTimeSensor(BaseSensorOperator): @@ -56,7 +55,6 @@ class DateTimeSensor(BaseSensorOperator): template_fields = ("target_time",) - @apply_defaults def __init__(self, *, target_time: Union[str, datetime.datetime], **kwargs) -> None: super().__init__(**kwargs) if isinstance(target_time, datetime.datetime): diff --git a/airflow/sensors/external_task.py b/airflow/sensors/external_task.py index 20b6f9086b676..929dcfd16a993 100644 --- a/airflow/sensors/external_task.py +++ b/airflow/sensors/external_task.py @@ -26,7 +26,6 @@ from airflow.models import BaseOperatorLink, DagBag, DagModel, DagRun, TaskInstance from airflow.operators.dummy import DummyOperator from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults from airflow.utils.helpers import build_airflow_url_with_query from airflow.utils.session import provide_session from airflow.utils.state import State @@ -87,7 +86,6 @@ def operator_extra_links(self): """Return operator extra links""" return [ExternalTaskSensorLink()] - @apply_defaults def __init__( self, *, @@ -273,7 +271,6 @@ class ExternalTaskMarker(DummyOperator): # The _serialized_fields are lazily loaded when get_serialized_fields() method is called __serialized_fields: Optional[FrozenSet[str]] = None - @apply_defaults def __init__( self, *, diff --git a/airflow/sensors/filesystem.py b/airflow/sensors/filesystem.py index c39b85df8c2b7..33f937a0dc40f 100644 --- a/airflow/sensors/filesystem.py +++ b/airflow/sensors/filesystem.py @@ -23,7 +23,6 @@ from airflow.hooks.filesystem import FSHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class FileSensor(BaseSensorOperator): @@ -44,7 +43,6 @@ class FileSensor(BaseSensorOperator): template_fields = ('filepath',) ui_color = '#91818a' - @apply_defaults def __init__(self, *, filepath, fs_conn_id='fs_default', **kwargs): super().__init__(**kwargs) self.filepath = filepath diff --git a/airflow/sensors/python.py b/airflow/sensors/python.py index 26a0011b992a0..5780cd154d4e6 100644 --- a/airflow/sensors/python.py +++ b/airflow/sensors/python.py @@ -18,7 +18,6 @@ from typing import Callable, Dict, List, Optional from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults from airflow.utils.operator_helpers import determine_kwargs @@ -48,7 +47,6 @@ class PythonSensor(BaseSensorOperator): template_fields = ('templates_dict', 'op_args', 'op_kwargs') - @apply_defaults def __init__( self, *, diff --git a/airflow/sensors/smart_sensor.py b/airflow/sensors/smart_sensor.py index 3df7313c05e6e..6c17beb8912ad 100644 --- a/airflow/sensors/smart_sensor.py +++ b/airflow/sensors/smart_sensor.py @@ -31,7 +31,6 @@ from airflow.settings import LOGGING_CLASS_PATH from airflow.stats import Stats from airflow.utils import helpers, timezone -from airflow.utils.decorators import apply_defaults from airflow.utils.email import send_email from airflow.utils.log.logging_mixin import set_context from airflow.utils.module_loading import import_string @@ -106,11 +105,18 @@ def create_new_task_handler(): Create task log handler for a sensor work. :return: log handler """ + from airflow.utils.log.secrets_masker import _secrets_masker # noqa + handler_config_copy = {k: handler_config[k] for k in handler_config} + del handler_config_copy['filters'] + formatter_config_copy = {k: formatter_config[k] for k in formatter_config} handler = dictConfigurator.configure_handler(handler_config_copy) formatter = dictConfigurator.configure_formatter(formatter_config_copy) handler.setFormatter(formatter) + + # We want to share the _global_ filterer instance, not create a new one + handler.addFilter(_secrets_masker()) return handler def _get_sensor_logger(self, si): @@ -304,7 +310,6 @@ class SmartSensorOperator(BaseOperator, SkipMixin): ui_color = '#e6f1f2' - @apply_defaults def __init__( self, poke_interval=180, diff --git a/airflow/sensors/sql.py b/airflow/sensors/sql.py index bc7ecf452e870..8e0b878f98bfb 100644 --- a/airflow/sensors/sql.py +++ b/airflow/sensors/sql.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook from airflow.sensors.base import BaseSensorOperator -from airflow.utils.decorators import apply_defaults class SqlSensor(BaseSensorOperator): @@ -58,7 +57,6 @@ class SqlSensor(BaseSensorOperator): ) ui_color = '#7c7287' - @apply_defaults def __init__( self, *, conn_id, sql, parameters=None, success=None, failure=None, fail_on_empty=False, **kwargs ): diff --git a/airflow/sensors/time_delta.py b/airflow/sensors/time_delta.py index b41606eed9d74..027dde5c8dbfb 100644 --- a/airflow/sensors/time_delta.py +++ b/airflow/sensors/time_delta.py @@ -18,7 +18,6 @@ from airflow.sensors.base import BaseSensorOperator from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults class TimeDeltaSensor(BaseSensorOperator): @@ -32,7 +31,6 @@ class TimeDeltaSensor(BaseSensorOperator): :type delta: datetime.timedelta """ - @apply_defaults def __init__(self, *, delta, **kwargs): super().__init__(**kwargs) self.delta = delta diff --git a/airflow/sensors/time_sensor.py b/airflow/sensors/time_sensor.py index 3de43674b6380..c80a5ebf3b745 100644 --- a/airflow/sensors/time_sensor.py +++ b/airflow/sensors/time_sensor.py @@ -18,7 +18,6 @@ from airflow.sensors.base import BaseSensorOperator from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults class TimeSensor(BaseSensorOperator): @@ -29,7 +28,6 @@ class TimeSensor(BaseSensorOperator): :type target_time: datetime.time """ - @apply_defaults def __init__(self, *, target_time, **kwargs): super().__init__(**kwargs) self.target_time = target_time diff --git a/airflow/sensors/weekday.py b/airflow/sensors/weekday.py index ebc43466dcabd..afe1fa98c5013 100644 --- a/airflow/sensors/weekday.py +++ b/airflow/sensors/weekday.py @@ -18,7 +18,6 @@ from airflow.sensors.base import BaseSensorOperator from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults from airflow.utils.weekday import WeekDay @@ -72,7 +71,6 @@ class DayOfWeekSensor(BaseSensorOperator): :type use_task_execution_day: bool """ - @apply_defaults def __init__(self, *, week_day, use_task_execution_day=False, **kwargs): super().__init__(**kwargs) self.week_day = week_day diff --git a/airflow/serialization/schema.json b/airflow/serialization/schema.json index 6803580c60753..e87c4aabf18e5 100644 --- a/airflow/serialization/schema.json +++ b/airflow/serialization/schema.json @@ -66,6 +66,12 @@ "maxProperties": 1 } }, + "dag_dependencies": { + "type": "array", + "items": { + "type": "object" + } + }, "dag": { "type": "object", "properties": { @@ -98,12 +104,14 @@ "is_paused_upon_creation": { "type": "boolean" }, "has_on_success_callback": { "type": "boolean" }, "has_on_failure_callback": { "type": "boolean" }, + "render_template_as_native_obj": { "type": "boolean" }, "tags": { "type": "array" }, "_task_group": {"anyOf": [ { "type": "null" }, { "$ref": "#/definitions/task_group" } ]}, - "edge_info": { "$ref": "#/definitions/edge_info" } + "edge_info": { "$ref": "#/definitions/edge_info" }, + "dag_dependencies": { "$ref": "#/definitions/dag_dependencies" } }, "required": [ "_dag_id", diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index 8a6fdc89d2062..9059728c9352b 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -19,6 +19,7 @@ import datetime import enum import logging +from dataclasses import dataclass from inspect import Parameter, signature from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Set, Union @@ -72,6 +73,15 @@ "airflow.sensors.external_task_sensor.ExternalTaskSensorLink", } +BUILTIN_OPERATOR_EXTRA_LINKS: List[str] = [ + "airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleLink", + "airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink", + "airflow.providers.google.cloud.operators.dataproc.DataprocJobLink", + "airflow.providers.google.cloud.operators.dataproc.DataprocClusterLink", + "airflow.providers.google.cloud.operators.mlengine.AIPlatformConsoleLink", + "airflow.providers.qubole.operators.qubole.QDSLink", +] + @cache def get_operator_extra_links(): @@ -293,8 +303,7 @@ def _deserialize(cls, encoded_var: Any) -> Any: # pylint: disable=too-many-retu elif type_ == DAT.SET: return {cls._deserialize(v) for v in var} elif type_ == DAT.TUPLE: - # pylint: disable=consider-using-generator - return tuple([cls._deserialize(v) for v in var]) + return tuple(cls._deserialize(v) for v in var) else: raise TypeError(f'Invalid type {type_!s} in deserialization.') @@ -335,6 +344,30 @@ def _value_is_hardcoded_default(cls, attrname: str, value: Any, instance: Any) - return False +class DependencyDetector: + """Detects dependencies between DAGs.""" + + @staticmethod + def detect_task_dependencies(task: BaseOperator) -> Optional['DagDependency']: + """Detects dependencies caused by tasks""" + if task.task_type == "TriggerDagRunOperator": + return DagDependency( + source=task.dag_id, + target=getattr(task, "trigger_dag_id"), + dependency_type="trigger", + dependency_id=task.task_id, + ) + elif task.task_type == "ExternalTaskSensor": + return DagDependency( + source=getattr(task, "external_dag_id"), + target=task.dag_id, + dependency_type="sensor", + dependency_id=task.task_id, + ) + + return None + + class SerializedBaseOperator(BaseOperator, BaseSerialization): """A JSON serializable representation of operator. @@ -350,6 +383,8 @@ class SerializedBaseOperator(BaseOperator, BaseSerialization): if v.default is not v.empty } + dependency_detector = DependencyDetector + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # task_type is used by UI to display the correct class type, because UI only @@ -494,6 +529,11 @@ def deserialize_operator(cls, encoded_op: Dict[str, Any]) -> BaseOperator: return op + @classmethod + def detect_dependencies(cls, op: BaseOperator) -> Optional['DagDependency']: + """Detects between DAG dependencies for the operator.""" + return cls.dependency_detector.detect_task_dependencies(op) + @classmethod def _is_excluded(cls, var: Any, attrname: str, op: BaseOperator): if var is not None and op.has_dag() and attrname.endswith("_date"): @@ -648,6 +688,11 @@ def serialize_dag(cls, dag: DAG) -> dict: serialize_dag = cls.serialize_to_json(dag, cls._decorated_fields) serialize_dag["tasks"] = [cls._serialize(task) for _, task in dag.task_dict.items()] + serialize_dag["dag_dependencies"] = [ + vars(t) + for t in (SerializedBaseOperator.detect_dependencies(task) for task in dag.task_dict.values()) + if t is not None + ] serialize_dag['_task_group'] = SerializedTaskGroup.serialize_task_group(dag.task_group) # Edge info in the JSON exactly matches our internal structure @@ -812,3 +857,20 @@ def deserialize_task_group( group.upstream_task_ids = set(cls._deserialize(encoded_group["upstream_task_ids"])) group.downstream_task_ids = set(cls._deserialize(encoded_group["downstream_task_ids"])) return group + + +@dataclass +class DagDependency: + """Dataclass for representing dependencies between DAGs. + These are calculated during serialization and attached to serialized DAGs. + """ + + source: str + target: str + dependency_type: str + dependency_id: str + + @property + def node_id(self): + """Node ID for graph rendering""" + return f"{self.dependency_type}:{self.source}:{self.target}:{self.dependency_id}" diff --git a/airflow/settings.py b/airflow/settings.py index bf68020e4199a..580aa97c6b991 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -205,6 +205,8 @@ def configure_vars(): def configure_orm(disable_connection_pool=False): """Configure ORM using SQLAlchemy""" + from airflow.utils.log.secrets_masker import mask_secret + log.debug("Setting up DB connection pool (PID %s)", os.getpid()) global engine global Session @@ -220,6 +222,9 @@ def configure_orm(disable_connection_pool=False): connect_args = {} engine = create_engine(SQL_ALCHEMY_CONN, connect_args=connect_args, **engine_args) + + mask_secret(engine.url.password) + setup_event_handlers(engine) Session = scoped_session( @@ -239,7 +244,7 @@ def prepare_engine_args(disable_connection_pool=False): if disable_connection_pool or not pool_connections: engine_args['poolclass'] = NullPool log.debug("settings.prepare_engine_args(): Using NullPool") - elif 'sqlite' not in SQL_ALCHEMY_CONN: + elif not SQL_ALCHEMY_CONN.startswith('sqlite'): # Pool size engine args not supported by sqlite. # If no config value is defined for the pool size, select a reasonable value. # 0 means no limit, which could lead to exceeding the Database connection limit. @@ -282,6 +287,15 @@ def prepare_engine_args(disable_connection_pool=False): engine_args['pool_recycle'] = pool_recycle engine_args['pool_pre_ping'] = pool_pre_ping engine_args['max_overflow'] = max_overflow + + # The default isolation level for MySQL (REPEATABLE READ) can introduce inconsistencies when + # running multiple schedulers, as repeated queries on the same session may read from stale snapshots. + # 'READ COMMITTED' is the default value for PostgreSQL. + # More information here: + # https://dev.mysql.com/doc/refman/8.0/en/innodb-transaction-isolation-levels.html" + if SQL_ALCHEMY_CONN.startswith('mysql'): + engine_args['isolation_level'] = 'READ COMMITTED' + return engine_args @@ -497,3 +511,9 @@ def initialize(): executor_constants.KUBERNETES_EXECUTOR, executor_constants.CELERY_KUBERNETES_EXECUTOR, } + +HIDE_SENSITIVE_VAR_CONN_FIELDS = conf.getboolean('core', 'hide_sensitive_var_conn_fields') + +# By default this is off, but is automatically configured on when running task +# instances +MASK_SECRETS_IN_LOGS = False diff --git a/airflow/typing_compat.py b/airflow/typing_compat.py index 6fd6d8c8252f2..d98eb7b5562a0 100644 --- a/airflow/typing_compat.py +++ b/airflow/typing_compat.py @@ -32,3 +32,12 @@ ) except ImportError: from typing_extensions import Protocol, TypedDict, runtime_checkable # type: ignore # noqa + + +# Before Py 3.7, there is no re.Pattern class +try: + from re import Pattern as RePatternType # type: ignore # pylint: disable=unused-import +except ImportError: + import re + + RePatternType = type(re.compile('', 0)) diff --git a/airflow/ui/docs/CONTRIBUTING.md b/airflow/ui/docs/CONTRIBUTING.md index 549d33a6f2f66..21469925b5a81 100644 --- a/airflow/ui/docs/CONTRIBUTING.md +++ b/airflow/ui/docs/CONTRIBUTING.md @@ -23,7 +23,7 @@ If you're new to modern frontend development or parts of our stack, you may want to check out these resources to understand our codebase: -- Typescript is an extension of javascript to add type-checking to our app. Files ending in `.ts` or `.tsx` will be type-checked. Check out the [handbook](https://www.typescriptlang.org/docs/handbook/typescript-in-5-minutes-func.html) for an introduction or feel free to keep this [cheatsheet](https://github.com/typescript-cheatsheets/react) open while developing. +- TypeScript is an extension of javascript to add type-checking to our app. Files ending in `.ts` or `.tsx` will be type-checked. Check out the [handbook](https://www.typescriptlang.org/docs/handbook/typescript-in-5-minutes-func.html) for an introduction or feel free to keep this [cheatsheet](https://github.com/typescript-cheatsheets/react) open while developing. - React powers our entire app so it would be valuable to learn JSX, the html-in-js templates React utilizes. Files that contain JSX will end in `.tsx` instead of `.ts`. Check out their official [tutorial](https://reactjs.org/tutorial/tutorial.html#overview) for a basic overview. @@ -42,7 +42,7 @@ the more confidence they can give you." Keep their [cheatsheet](https://testing- - `.neutrinorc.js` is the main config file. Although some custom typescript or linting may need to be changed in `tsconfig.json` or `.eslintrc.js`, respectively - `src/components` are React components that will be shared across the app - `src/views` are React components that are specific to a certain url route -- `src/interfaces` are custom-defined Typescript types/interfaces +- `src/interfaces` are custom-defined TypeScript types/interfaces - `src/utils` contains various helper functions that are shared throughout the app - `src/auth` has the Context for authentication - `src/api` contains all of the actual API requests as custom hooks around react-query diff --git a/airflow/ui/package.json b/airflow/ui/package.json index 8788aa2f25425..d7e7857b53892 100644 --- a/airflow/ui/package.json +++ b/airflow/ui/package.json @@ -9,10 +9,11 @@ "lint": "eslint --format codeframe --ext mjs,jsx,js,tsx,ts src test && tsc" }, "dependencies": { - "@chakra-ui/react": "^1.3.4", + "@chakra-ui/react": "^1.6.1", "@emotion/react": "^11.1.5", "@emotion/styled": "^11.1.5", "@neutrinojs/copy": "^9.5.0", + "@vvo/tzdb": "^6.7.0", "axios": "^0.21.1", "dayjs": "^1.10.4", "dotenv": "^8.2.0", @@ -24,6 +25,7 @@ "react-icons": "^4.2.0", "react-query": "^3.12.3", "react-router-dom": "^5.2.0", + "react-select": "^4.3.0", "use-react-router": "^1.0.7" }, "devDependencies": { @@ -37,6 +39,7 @@ "@types/react": "^17.0.3", "@types/react-dom": "^17.0.2", "@types/react-router-dom": "^5.1.7", + "@types/react-select": "^4.0.15", "eslint": "^7", "eslint-config-airbnb-typescript": "^12.3.1", "eslint-plugin-import": "^2.22.1", diff --git a/airflow/ui/src/components/AppContainer/AppHeader.tsx b/airflow/ui/src/components/AppContainer/AppHeader.tsx index f5beda0d3d3ec..cf2bb90b4d2a1 100644 --- a/airflow/ui/src/components/AppContainer/AppHeader.tsx +++ b/airflow/ui/src/components/AppContainer/AppHeader.tsx @@ -21,8 +21,6 @@ import React from 'react'; import { Link } from 'react-router-dom'; import { Avatar, - Box, - Button, Flex, Icon, Menu, @@ -32,7 +30,6 @@ import { MenuItem, useColorMode, useColorModeValue, - Tooltip, } from '@chakra-ui/react'; import { MdWbSunny, @@ -41,13 +38,14 @@ import { MdExitToApp, } from 'react-icons/md'; import dayjs from 'dayjs'; -import timezone from 'dayjs/plugin/timezone'; +import tz from 'dayjs/plugin/timezone'; import { useAuthContext } from 'providers/auth/context'; import ApacheAirflowLogo from 'components/icons/ApacheAirflowLogo'; +import TimezoneDropdown from './TimezoneDropdown'; -dayjs.extend(timezone); +dayjs.extend(tz); interface Props { bodyBg: string; @@ -57,14 +55,11 @@ interface Props { const AppHeader: React.FC = ({ bodyBg, overlayBg, breadcrumb }) => { const { toggleColorMode } = useColorMode(); - const now = dayjs().tz(); const headerHeight = '56px'; const { hasValidAuthToken, logout } = useAuthContext(); const darkLightIcon = useColorModeValue(MdBrightness2, MdWbSunny); const darkLightText = useColorModeValue(' Dark ', ' Light '); - const handleOpenTZ = () => window.alert('This will open time zone select modal!'); - const handleOpenProfile = () => window.alert('This will take you to your user profile view.'); return ( @@ -91,18 +86,7 @@ const AppHeader: React.FC = ({ bodyBg, overlayBg, breadcrumb }) => { )} {hasValidAuthToken && ( - - {/* TODO: open modal for time zone update */} - - + diff --git a/airflow/ui/src/components/AppContainer/TimezoneDropdown.tsx b/airflow/ui/src/components/AppContainer/TimezoneDropdown.tsx new file mode 100644 index 0000000000000..aae91cb030395 --- /dev/null +++ b/airflow/ui/src/components/AppContainer/TimezoneDropdown.tsx @@ -0,0 +1,89 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React, { useState, useRef } from 'react'; +import { + Box, + Button, + Menu, + MenuButton, + MenuList, + Tooltip, +} from '@chakra-ui/react'; +import dayjs from 'dayjs'; +import tz from 'dayjs/plugin/timezone'; +import { getTimeZones } from '@vvo/tzdb'; + +import Select from 'components/MultiSelect'; +import { useTimezoneContext } from 'providers/TimezoneProvider'; + +dayjs.extend(tz); + +interface Option { value: string, label: string } + +const TimezoneDropdown: React.FC = () => { + const { timezone, setTimezone } = useTimezoneContext(); + const [now, setNow] = useState(dayjs().tz(timezone)); + const menuRef = useRef(null); + + const timezones = getTimeZones(); + + let currentTimezone; + const options = timezones.map(({ name, currentTimeFormat, group }) => { + const label = `${currentTimeFormat.substring(0, 6)} ${name.replace(/_/g, ' ')}`; + if (name === timezone || group.includes(timezone)) currentTimezone = { label, value: name }; + return { label, value: name }; + }); + + const onChangeTimezone = (newTimezone: Option | null) => { + if (newTimezone) { + setTimezone(newTimezone.value); + setNow(dayjs().tz(newTimezone.value)); + // Close the dropdown on a successful change + menuRef?.current?.click(); + } + }; + + return ( + + + + + {now.format('HH:mm Z')} + + + + + { + const inputStyles = useMultiStyleConfig('Input', {}); + return ( + + + {children} + + + ); + }, + MultiValueContainer: ({ + children, + innerRef, + innerProps, + data: { isFixed }, + }) => ( + + {children} + + ), + MultiValueLabel: ({ children, innerRef, innerProps }) => ( + + {children} + + ), + MultiValueRemove: ({ + children, innerRef, innerProps, data: { isFixed }, + }) => { + if (isFixed) { + return null; + } + + return ( + + {children} + + ); + }, + IndicatorSeparator: ({ innerProps }) => ( + + ), + ClearIndicator: ({ innerProps }) => ( + + ), + DropdownIndicator: ({ innerProps }) => { + const { addon } = useStyles(); + + return ( +
+ +
+ ); + }, + // Menu components + MenuPortal: ({ children, ...portalProps }) => ( + + {children} + + ), + Menu: ({ children, ...menuProps }) => { + const menuStyles = useMultiStyleConfig('Menu', {}); + return ( + + {children} + + ); + }, + MenuList: ({ + innerRef, children, maxHeight, + }) => { + const { list } = useStyles(); + return ( + + {children} + + ); + }, + GroupHeading: ({ innerProps, children }) => { + const { groupTitle } = useStyles(); + return ( + + {children} + + ); + }, + Option: ({ + innerRef, innerProps, children, isFocused, isDisabled, + }) => { + const { item } = useStyles(); + interface ItemProps extends CSSWithMultiValues { + _disabled: CSSWithMultiValues, + _focus: CSSWithMultiValues, + } + return ( + )._focus.bg : 'transparent', + ...(isDisabled && (item as RecursiveCSSObject)._disabled), + }} + ref={innerRef} + {...innerProps} + {...(isDisabled && { disabled: true })} + > + {children} + + ); + }, + }, + ...components, + }} + styles={{ + ...chakraStyles, + ...styles, + }} + theme={(baseTheme) => ({ + ...baseTheme, + borderRadius: chakraTheme.radii.md, + colors: { + ...baseTheme.colors, + neutral50: placeholderColor, // placeholder text color + neutral40: placeholderColor, // noOptionsMessage color + }, + })} + {...props} + /> + ); +}; + +export default MultiSelect; diff --git a/airflow/ui/src/providers/TimezoneProvider.tsx b/airflow/ui/src/providers/TimezoneProvider.tsx index 4864702f263df..193e8ce557860 100644 --- a/airflow/ui/src/providers/TimezoneProvider.tsx +++ b/airflow/ui/src/providers/TimezoneProvider.tsx @@ -45,9 +45,9 @@ type Props = { const TimezoneProvider = ({ children }: Props): ReactElement => { // TODO: add in default_timezone when GET /ui-metadata is available - // guess timezone on browser or default to utc - const [timezone, setTimezone] = useState(dayjs.tz.guess() || 'UTC'); - + // guess timezone on browser or default to utc and don't guess when testing + const isTest = process.env.NODE_ENV === 'test'; + const [timezone, setTimezone] = useState(isTest ? 'UTC' : dayjs.tz.guess()); useEffect(() => { dayjs.tz.setDefault(timezone); }, [timezone]); diff --git a/airflow/ui/src/views/Pipelines/PipelinesTable.tsx b/airflow/ui/src/views/Pipelines/PipelinesTable.tsx new file mode 100644 index 0000000000000..8b1a4daa98515 --- /dev/null +++ b/airflow/ui/src/views/Pipelines/PipelinesTable.tsx @@ -0,0 +1,77 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import { + Alert, + AlertIcon, + Table, + Thead, + Tbody, + Tr, + Th, + Td, +} from '@chakra-ui/react'; + +import { defaultDags } from 'api/defaults'; +import { useDags } from 'api'; +import type { Dag } from 'interfaces'; +import Row from './Row'; + +const PipelinesTable: React.FC = () => { + const { data: { dags } = defaultDags, isLoading, error } = useDags(); + + return ( + <> + {error && ( + + + {error.message} + + )} + + + + + + + + {isLoading && ( + + + + )} + {(!isLoading && !dags.length) && ( + + + + )} + {dags.map((dag: Dag) => )} + +
+ DAG ID +
Loading…
No Pipelines found.
+ + ); +}; + +export default PipelinesTable; diff --git a/airflow/ui/src/views/Pipelines/index.tsx b/airflow/ui/src/views/Pipelines/index.tsx index c5fbcdcd5c0bf..13cfc6b356f18 100644 --- a/airflow/ui/src/views/Pipelines/index.tsx +++ b/airflow/ui/src/views/Pipelines/index.tsx @@ -18,61 +18,15 @@ */ import React from 'react'; -import { - Alert, - AlertIcon, - Table, - Thead, - Tbody, - Tr, - Th, - Td, -} from '@chakra-ui/react'; import AppContainer from 'components/AppContainer'; -import { defaultDags } from 'api/defaults'; -import { useDags } from 'api'; -import type { Dag } from 'interfaces'; -import Row from './Row'; +import PipelinesTable from './PipelinesTable'; -const Pipelines: React.FC = () => { - const { data: { dags } = defaultDags, isLoading, error } = useDags(); - - return ( - - {error && ( - - - {error.message} - - )} - - - - - - - - {isLoading && ( - - - - )} - {(!isLoading && !dags.length) && ( - - - - )} - {dags.map((dag: Dag) => )} - -
- DAG ID -
Loading…
No Pipelines found.
- - ); -}; +// A separate PipelinesTable component limits how much is rerendered on data refetch +const Pipelines: React.FC = () => ( + + + +); export default Pipelines; diff --git a/airflow/ui/test/TimezoneDropdown.test.tsx b/airflow/ui/test/TimezoneDropdown.test.tsx new file mode 100644 index 0000000000000..66051e6416c64 --- /dev/null +++ b/airflow/ui/test/TimezoneDropdown.test.tsx @@ -0,0 +1,59 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import '@testing-library/jest-dom'; +import { render, fireEvent } from '@testing-library/react'; + +import dayjs from 'dayjs'; +import timezone from 'dayjs/plugin/timezone'; +import utc from 'dayjs/plugin/utc'; + +import TimezoneDropdown from 'components/AppContainer/TimezoneDropdown'; +import TimezoneProvider from 'providers/TimezoneProvider'; +import { ChakraWrapper } from './utils'; + +dayjs.extend(utc); +dayjs.extend(timezone); + +describe('test timezone dropdown', () => { + test('Can search for a new timezone and the date changes', () => { + const { getByText } = render( + + + , + { wrapper: ChakraWrapper }, + ); + + const initialTime = dayjs().tz('UTC').format('HH:mm Z'); + + expect(getByText(initialTime)).toBeInTheDocument(); + const button = getByText(initialTime); + fireEvent.click(button); + const focusedElement = document.activeElement; + if (focusedElement) { + fireEvent.change(focusedElement, { target: { value: 'Anch' } }); + } + const option = getByText('-08:00 America/Anchorage'); + expect(option).toBeInTheDocument(); + fireEvent.click(option); + + expect(getByText(dayjs().tz('America/Anchorage').format('HH:mm Z'))).toBeInTheDocument(); + }); +}); diff --git a/airflow/ui/test/utils.tsx b/airflow/ui/test/utils.tsx index df662b001cac5..e2d134710d218 100644 --- a/airflow/ui/test/utils.tsx +++ b/airflow/ui/test/utils.tsx @@ -17,10 +17,12 @@ * under the License. */ +import { ChakraProvider } from '@chakra-ui/react'; import { createMemoryHistory } from 'history'; import React from 'react'; import { QueryClient, QueryClientProvider } from 'react-query'; import { Router } from 'react-router-dom'; +import theme from 'theme'; export const url: string = `${process.env.WEBSERVER_URL}/api/v1/` || ''; @@ -44,3 +46,7 @@ export const RouterWrapper: React.FC<{}> = ({ children }) => { const history = createMemoryHistory(); return {children}; }; + +export const ChakraWrapper: React.FC<{}> = ({ children }) => ( + {children} +); diff --git a/airflow/ui/tsconfig.json b/airflow/ui/tsconfig.json index 85085f279cb82..eaa2d9039619b 100644 --- a/airflow/ui/tsconfig.json +++ b/airflow/ui/tsconfig.json @@ -1,5 +1,5 @@ /* -* Typescript config +* TypeScript config */ { "compilerOptions": { diff --git a/airflow/ui/yarn.lock b/airflow/ui/yarn.lock index 26dfd8793aa89..3e75fd01f540f 100644 --- a/airflow/ui/yarn.lock +++ b/airflow/ui/yarn.lock @@ -912,6 +912,13 @@ dependencies: regenerator-runtime "^0.13.4" +"@babel/runtime@^7.12.0", "@babel/runtime@^7.8.7": + version "7.14.0" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.14.0.tgz#46794bc20b612c5f75e62dd071e24dfd95f1cbe6" + integrity sha512-JELkvo/DlpNdJ7dlyw/eY7E0suy5i5GQH+Vlxaq1nsNJ+H7f4Vtv3jMeCEgRhZZQFXTjldYfQgv2qmM6M1v5wA== + dependencies: + regenerator-runtime "^0.13.4" + "@babel/template@^7.12.13", "@babel/template@^7.3.3": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.12.13.tgz#530265be8a2589dbb37523844c5bcb55947fb327" @@ -950,496 +957,541 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@chakra-ui/accordion@1.1.4": - version "1.1.4" - resolved "https://registry.yarnpkg.com/@chakra-ui/accordion/-/accordion-1.1.4.tgz#7f114ee546185a9cfc3fc0dfbbb63daaae2af8cb" - integrity sha512-8SBSZlxtGuZEi9iXeVF+V9ziM7RH1MW1WJ0K6AcQQVU8uyMa6Ld+FWzsROawwkEJgVCrrd3tFH9coY/0ZXlZRQ== +"@chakra-ui/accordion@1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@chakra-ui/accordion/-/accordion-1.3.1.tgz#86d7cb4a1304fdb4031aebb287290df285f90684" + integrity sha512-BunY+Ex6rfES8E28+cusFrNlh3MTquhs0OSAZw8eEHeg0586bR/QIizRh5Bw+S5aineF1wKppsVUMYEJFc5SlQ== dependencies: - "@chakra-ui/descendant" "1.0.9" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/icon" "1.1.3" - "@chakra-ui/transition" "1.1.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/descendant" "2.0.1" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/icon" "1.1.9" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/transition" "1.3.0" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/alert@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/alert/-/alert-1.1.3.tgz#05b86d62b4a20d5e0b4c88be788a79cd5666e071" - integrity sha512-Q/A2K6m/FqN4W/bNh+OAPzrwNGGGS9yxznPTQMHC0Map34t0/pP2A/luKC00B0oDHhF9AAcLosjn3NXhphT2pw== +"@chakra-ui/alert@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/alert/-/alert-1.2.5.tgz#d8ab0fdf75fa61625c368bd1979d2989a9fa1907" + integrity sha512-S+YaqaACteSOhGlBFygOeYsQC2FYO6m37g6M+N23kbUHKtT7I4gYsxczLuUwKJ65u4T2Bt1qNSVokjmKCMhE9w== dependencies: - "@chakra-ui/icon" "1.1.3" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/icon" "1.1.9" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/avatar@1.1.4": - version "1.1.4" - resolved "https://registry.yarnpkg.com/@chakra-ui/avatar/-/avatar-1.1.4.tgz#63a41624fabee5cad35380f4cd1ced3bc8a4d413" - integrity sha512-/l7ibd8at75gIW3WzU9cis0+o5lWfwgUO4oI9yevFgllSyyPgwgZhdR/lcTQmJlmEuA0eaawQPM3GN8LsJfFnQ== +"@chakra-ui/avatar@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/avatar/-/avatar-1.2.5.tgz#c11026a9e9e977d43e92212f0b27f43c52d42bd5" + integrity sha512-+BiCKZO/+oM7C5wYGKWttRQZTapVH6RSXxqhASgFBN1iz7H0mH1JtSSib7eBNKXZLM+RQcGrLPJBjQnc2TLVKA== dependencies: - "@chakra-ui/image" "1.0.9" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/image" "1.0.15" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/breadcrumb@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/breadcrumb/-/breadcrumb-1.1.3.tgz#956f58ac2e349a51dae47cad8139df2534c6ce3e" - integrity sha512-FpHlrJiheSQ6SAXvvZMp7juMOnaYeGFtmz4FK1n4sDaaim6jy7haMxbxY6yINOhMC/FGjpdpLLjdK0lPKUuZPw== +"@chakra-ui/breadcrumb@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/breadcrumb/-/breadcrumb-1.2.5.tgz#e8eac928df24b6ec06945b343f6a74551bf862f4" + integrity sha512-cD68feXlxyKkYqXbdWcc/Y78pLf8tFaHf6Vc5lTcgEYAxav5vq8TZqhzDwqMS84mOvuD6kd1wUbyBHff5h2Czw== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/button@1.1.4": - version "1.1.4" - resolved "https://registry.yarnpkg.com/@chakra-ui/button/-/button-1.1.4.tgz#6b474f56f51d46115894342a8a06a2f5321faedc" - integrity sha512-xLrISCqC/oe+QuBfhfTmdxCNbgad2dz5lu2BZ9TE6d1ppeP13RtFArQHaJ9/0wUVs3Oke6+ksSr1oCvGXvDadA== +"@chakra-ui/button@1.3.2": + version "1.3.2" + resolved "https://registry.yarnpkg.com/@chakra-ui/button/-/button-1.3.2.tgz#7a119ec94795d130375c1440d2dbcc9d768bcd50" + integrity sha512-ED23SD7smL9MdfkQE4CK+X6L7vRvhWwSsopuppqV8oQV+iktjY+lAMDIGlz9w7FPkNpv0q68yZ++qxPZRRv8xw== dependencies: - "@chakra-ui/spinner" "1.1.3" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/spinner" "1.1.9" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/checkbox@1.3.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/checkbox/-/checkbox-1.3.0.tgz#51874dcdbc038ac2d896ddd9c327f966619a3353" - integrity sha512-erZxnK7ybI7ruyAQaDaDzPEV4eL+AJgP0dWQBzssLwOUhLqYkLG5km/tmgZZzfwg5A62WaBf9QT0tZymWTJ+ng== +"@chakra-ui/checkbox@1.5.2": + version "1.5.2" + resolved "https://registry.yarnpkg.com/@chakra-ui/checkbox/-/checkbox-1.5.2.tgz#a90240126d51ade0551de7f66c1f66bc6ed191cb" + integrity sha512-o1uuQOY87N6xKGGUB+2Bh6qREwYch2Jl6rIc+/zhVogEEfBJ5M2lCqmQNr+hepsD2Nw466/AjQM+dNuPHuWfhg== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" - "@chakra-ui/visually-hidden" "1.0.6" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" + "@chakra-ui/visually-hidden" "1.0.12" -"@chakra-ui/clickable@1.0.6": - version "1.0.6" - resolved "https://registry.yarnpkg.com/@chakra-ui/clickable/-/clickable-1.0.6.tgz#9ab45fbbfe01ac484f5a3d6f5c26e13000353d52" - integrity sha512-k4+mS2l3GbYbJbdIVpZzAB3xQ9EiECUoMmlZKp+NHh9pUR2XZWSmjVhfedVyk7WuOfpXWxAhwhgCVrM5+hRd9w== +"@chakra-ui/clickable@1.1.5": + version "1.1.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/clickable/-/clickable-1.1.5.tgz#21c439aee65880a133c91413c08dcf17095890ab" + integrity sha512-nC6vLxh7cYrYyuMdrotiq+CUqG+KGND+RLeE3WiWyP0JDivX6t7S1EtRENnskKyWaLvjMDFsLU0wDAZm18vQNA== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/close-button@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/close-button/-/close-button-1.1.3.tgz#db0a0c62549a14c529deae8184d20466170fe3e8" - integrity sha512-pLj5E2or8VdQ69w/ZZ/B09E446B6wbmJFQ8fuqnaxOeBZKp3VtvOLXWDg23SsijKKgw71EBq0O7qUOKvQRygKA== +"@chakra-ui/close-button@1.1.9": + version "1.1.9" + resolved "https://registry.yarnpkg.com/@chakra-ui/close-button/-/close-button-1.1.9.tgz#f6f0aba2de0e15881e4ec20dd1ba2e7d6cac4630" + integrity sha512-Wj1nuNUw93AkswwQM+ifOi0guJddRavu4mJS4F15+Vv8d30BQo/O6NOGPFutJK4Fm48WA9QjqydCMb1n8umDxA== dependencies: - "@chakra-ui/icon" "1.1.3" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/icon" "1.1.9" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/color-mode@1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@chakra-ui/color-mode/-/color-mode-1.1.2.tgz#824ea170b04152021bc5c39a7f17f7f0b6d59801" - integrity sha512-RnJvLFNmLyC9bnYgzunJ19pwr7chc3T6mvZJG4Goqxdn8jPQ3QJPh6A0vA5tvIIgbXqvIPn8imknBntHq1P0kw== +"@chakra-ui/color-mode@1.1.8": + version "1.1.8" + resolved "https://registry.yarnpkg.com/@chakra-ui/color-mode/-/color-mode-1.1.8.tgz#a5648c017868d22947827009e5ea4f0cd0d0ad87" + integrity sha512-hmSK02Eozu42g1yaQIXfJg+8Dag/YoeaQLS8Ygk6OEGEZrILMxLbFEBvzxaCJt5GCKOuVc/9aGsn31N7c4SNrA== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/control-box@1.0.6": - version "1.0.6" - resolved "https://registry.yarnpkg.com/@chakra-ui/control-box/-/control-box-1.0.6.tgz#aa6f2b385fb22cf2033e1e567db87321a70bfb6e" - integrity sha512-5Tkwre9BF3wifx6pe5cgBlpcAZaJWhLF2KGs29lsmjOHV14JvLff9sPNaqNIaugRZY25bF4JVgMXg42aIfS17Q== +"@chakra-ui/control-box@1.0.12": + version "1.0.12" + resolved "https://registry.yarnpkg.com/@chakra-ui/control-box/-/control-box-1.0.12.tgz#2038d2f62d0640342930b19ee4b91b955cd4c575" + integrity sha512-CZCqrm0rMj72/9i5raaKgxdJm0uGdES+8p9TdVesfSXZnsNmOquDY9TsI5fYMhG16yRRoiQK/dmX/j2VcD7tvA== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/counter@1.0.9": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@chakra-ui/counter/-/counter-1.0.9.tgz#e0a107293c433856e17de8ec6d755b891fc60e9f" - integrity sha512-D1iXgA2rjHhZx2Gm3Tp3BD6fCX+LMvI0D/SkWGnJ5ES4n/0QXk1AoEAaz7jIrk6nEY1Sv/U4GxErRTV3KtSOzw== +"@chakra-ui/counter@1.1.5": + version "1.1.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/counter/-/counter-1.1.5.tgz#9355a231bb0929e22627cf3d51a9de0d782002bf" + integrity sha512-qKUaQgCXWutLpEYz0fuBOQz+tfk4Lt8EN+ePikgxN6LPoisw/1Nbo+HE7l1A9YQ06wUG0HBt7s8CcupT/LzRug== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/utils" "1.8.0" "@chakra-ui/css-reset@1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@chakra-ui/css-reset/-/css-reset-1.0.0.tgz#8395921b35ef27bee0579a4d730c5ab7f7b39734" integrity sha512-UaPsImGHvCgFO3ayp6Ugafu2/3/EG8wlW/8Y9Ihfk1UFv8cpV+3BfWKmuZ7IcmxcBL9dkP6E8p3/M1T0FB92hg== -"@chakra-ui/descendant@1.0.9": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@chakra-ui/descendant/-/descendant-1.0.9.tgz#acc0470c42d5ec962586a91e7165c8e654377f5f" - integrity sha512-N/s5+mr7SfDHdMDKcXaJsISYxtIoUTy+Wj9fYbVO2ABx5TrMWc/6Y+3sIlEHzobpsAGfaPTLUhTUYT9P82zGSw== +"@chakra-ui/descendant@2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@chakra-ui/descendant/-/descendant-2.0.1.tgz#fc3bc9081aa01503035b2c9296bc4b9f87ceaae0" + integrity sha512-TeYp94iOhu5Gs2oVzewJaep0qft/JKMKfmcf4PGgzJF+h6TWZm6NGohk6Jq7JOh+y0rExa1ulknIgnMzFx5xaA== dependencies: - "@chakra-ui/hooks" "1.2.0" + "@chakra-ui/react-utils" "^1.1.2" -"@chakra-ui/editable@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/editable/-/editable-1.1.3.tgz#7abaa3ebc5bcd2402d2a39d47948c437aa959eae" - integrity sha512-jH38V+LQveXAxB80GAw5kXkB4Q0BbV+1YU/u5fyxYjoQ2GSkTe34kUiWXzCtYdnHska8xaZ08qtkfEcS5eDZsQ== +"@chakra-ui/editable@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/editable/-/editable-1.2.5.tgz#ec02ab8e2ec47612ef105476dfbf1bbdf8b62ea5" + integrity sha512-rSsBj/X2Ta0tTFm61TSgI2wsrhWydlEPROcDAcV8IB4UTOFyCf/fSKDMPN20qj0gLh6OV52d4JTSwgeLpFluRQ== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/focus-lock@1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@chakra-ui/focus-lock/-/focus-lock-1.1.2.tgz#2e8fda42887372b39b440c1b90b9fa380dcc0376" - integrity sha512-t9CYCX+E/9rqVCpFq7emIWMkTYPSl5fBJtuTUoWBNVXeyDBHB+Kfg7miM2ET0Re2By4ZFTCofcCs5sTYHGbeQA== +"@chakra-ui/focus-lock@1.1.8": + version "1.1.8" + resolved "https://registry.yarnpkg.com/@chakra-ui/focus-lock/-/focus-lock-1.1.8.tgz#30139cc8ccc68038e3c148a5f5e57eccfdfcb54e" + integrity sha512-y2aZGFv5O8vWL/oxt3FhhQSD6lQO5TEUdIFwaKFCh1qqHMoWrQMx0n7aGny1E8IMGvs2fKMKr1lIA3HovE/sCw== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" react-focus-lock "2.5.0" -"@chakra-ui/form-control@1.2.3": - version "1.2.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/form-control/-/form-control-1.2.3.tgz#8ee931d769c22afbcdf62e304751de9a440130f2" - integrity sha512-FwT+Bet7wHg4yPZBjcJd08TVtHMgVc33iNT1gUKluMskHH1dPiXvgH0+bNLlQcjO3869qP6rqG0ul4BuWga/XA== +"@chakra-ui/form-control@1.3.5": + version "1.3.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/form-control/-/form-control-1.3.5.tgz#dc7514f966e4e2cc083d139bf268efbeb027e935" + integrity sha512-gAOWGyWzHp0IFHnHfeWUBFio7dsJvcpbFDguX+RlKZBIGYA7bjRTX8HRXW/T32sYCJn+2A3QJdh/oZjegk0+Xg== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/icon" "1.1.3" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/icon" "1.1.9" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/hooks@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/hooks/-/hooks-1.2.0.tgz#4faedf2840c9c8bfad6dac289cdd13ffbb36e6a5" - integrity sha512-un6FuNRVtX4YKT842otuthOPvsBMsRfiDFSu0afoOZHcxfUPmtPFdE0mWQGlyAcolaj5wkL7zxXR6tbY47FnBA== +"@chakra-ui/hooks@1.5.2": + version "1.5.2" + resolved "https://registry.yarnpkg.com/@chakra-ui/hooks/-/hooks-1.5.2.tgz#9bde3c698790a0c94286701cad6ec2ce09c1484c" + integrity sha512-8rpZbrB3yFgKAjEwzafJkYOckRNYDQ6sSt0IWOFsfwGSGB3RxX19tKiGMziqU/djzKEGU85QErsVYVC1v0tJ3A== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" compute-scroll-into-view "1.0.14" copy-to-clipboard "3.3.1" -"@chakra-ui/icon@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/icon/-/icon-1.1.3.tgz#5fbc310652fa20da973ea9422ed5e1a654596d23" - integrity sha512-XZ9RTU0J/qB5bZawUK5yk6YN3P2fi74aTLap1qygmodAPo5sIroRzfuXndezjAYFhuRjI62zfSs/FVhwJyhmsw== +"@chakra-ui/icon@1.1.9": + version "1.1.9" + resolved "https://registry.yarnpkg.com/@chakra-ui/icon/-/icon-1.1.9.tgz#fbd6e82abf58b890f5bcc728bb75dda25d7b6c63" + integrity sha512-lmZHK4O+Wo7LwxRsQb0KmtOwq2iACESxwQgackuj7NPHbAsdF2/y3t2f7KCD+dTKGxoauEMgU2nVLePZWjtpTQ== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/image@1.0.9": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@chakra-ui/image/-/image-1.0.9.tgz#bd301f6028000d9dc3370c897868b465edc39ed5" - integrity sha512-GpfwgXzVC/IVkcFGUX0I4/NcQafjdCCSVNkdTRxYo0v368VrXb4087i2ypx+7w4PgRqZv3ABunM2BbbXE35U0w== +"@chakra-ui/image@1.0.15": + version "1.0.15" + resolved "https://registry.yarnpkg.com/@chakra-ui/image/-/image-1.0.15.tgz#57aaf7e22d2793b9278481aab56b27063a922e5b" + integrity sha512-bOmPspVXpgL70qJ5t1qxlAr4gLj3r1yulM3CByqj510mIZHt8vkh5zFu+8mN184V9U8Zw8fwL6W0FFg+ftmYCg== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/input@1.1.4": - version "1.1.4" - resolved "https://registry.yarnpkg.com/@chakra-ui/input/-/input-1.1.4.tgz#c4a7ada280df66fc0fc3e1fa1edb2bb4a4d286e0" - integrity sha512-dk2JD7tfRmQLqbQXpDLrkRjVpdZeVIqNKPqzQFOTApzQPxyF0/wHVbrPtipQCzlqwGuzhF0WDVc+G/KAQ+3SmA== - dependencies: - "@chakra-ui/form-control" "1.2.3" - "@chakra-ui/utils" "1.4.0" - -"@chakra-ui/layout@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/layout/-/layout-1.3.3.tgz#837d2d7be4d2c8f57b3aee2eb9288f740ffbb0d4" - integrity sha512-PvJSNAkTNmeYv10J0tD1oBZdKSeWDkrb0hrF8IrGtmLztL+gA5fGMIkNN4/bWq25mqI7mCRHqwYei4PhhZKYXg== +"@chakra-ui/input@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/input/-/input-1.2.5.tgz#f6e86d10f6e41bb0e721041840ce0fe122b6dc61" + integrity sha512-paDyXHYxI/zPqWpDcRzOX/RqNA+G0Z6Lto04xi1oJKJEItRggFwrAg2RgbhxmaMCMoT3LO47sJgwU9f6XCyVWQ== dependencies: - "@chakra-ui/icon" "1.1.3" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/form-control" "1.3.5" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/live-region@1.0.6": - version "1.0.6" - resolved "https://registry.yarnpkg.com/@chakra-ui/live-region/-/live-region-1.0.6.tgz#85d20b273d35a27ebe35e98cc9f6c9d66500e38e" - integrity sha512-pLtOXsb4H0+WNtxro3KGjaoTmruvXy7uedcf61lm3d3QaRAmA+R5nCjnGayvSGK9ex8Wr+CJQSzTUUijNzKiwg== +"@chakra-ui/layout@1.4.5": + version "1.4.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/layout/-/layout-1.4.5.tgz#89e6164a752a07b735f0a7a02437ae38ecada154" + integrity sha512-+9NczG/9vtditXHZdHSzBvr2k8Sfb59C6PQT1GaDVhwi693tpYDWZEc3vcin4XJbyRS5HNm8L9nwx6A6un/g5g== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/icon" "1.1.9" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/media-query@1.0.7": - version "1.0.7" - resolved "https://registry.yarnpkg.com/@chakra-ui/media-query/-/media-query-1.0.7.tgz#08d9011ef7a5dcb238026fd35549257e21f1fcac" - integrity sha512-loonqcxL4Buqt6c558D8fR6zwQiL4yRqTG3LZZMP3UtFo4ja9tyFI0bFtmnP1PYMgOrlITkty82KuLshppwuFg== +"@chakra-ui/live-region@1.0.12": + version "1.0.12" + resolved "https://registry.yarnpkg.com/@chakra-ui/live-region/-/live-region-1.0.12.tgz#ff9806c27f955eab5492fa89f186a7fb09c78b4a" + integrity sha512-NgIZuCLAX7FCyLc7+7wxQ0WCmlFS+SHutHf2QXezU6NjkoUmiZ02yVoBHltGn9f0jBVkVBZ4LL1ng2ruwRsJog== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/menu@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/menu/-/menu-1.2.0.tgz#489738547f69d2d36a67cb3863cb620b0454fd97" - integrity sha512-OYVlc1F2QNtCMWdrs611DnfqbLNTkKtpKUDJnaxGB8lzHY4aN0rPrYo0eIiY+10TCDLXCm7o4jWd+vHn1zqBGA== +"@chakra-ui/media-query@1.0.13": + version "1.0.13" + resolved "https://registry.yarnpkg.com/@chakra-ui/media-query/-/media-query-1.0.13.tgz#b6dd0c02481e38088fb9b21d6cfe279b335fc482" + integrity sha512-uNkZ9ci9sXB1J9uFg0vxukJjb2nC4KXWUZpnDbcpqOVEAAJx7iTrGs9+w9oJqtsTVlQEmx3U92EObifd9bYcWQ== dependencies: - "@chakra-ui/clickable" "1.0.6" - "@chakra-ui/descendant" "1.0.9" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/popper" "2.0.0" - "@chakra-ui/transition" "1.1.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/modal@1.7.0": - version "1.7.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/modal/-/modal-1.7.0.tgz#801ff8dbb27e4f740739768500973e6097536ba4" - integrity sha512-TxXcGbMdNkiD4WU55FLshAk4XlHvoUraKAJ1rcI5AcNc/DXuW8wy9P1t8Ho0iGgq1xRWQjzDgDfgBOh98fO7fQ== - dependencies: - "@chakra-ui/close-button" "1.1.3" - "@chakra-ui/focus-lock" "1.1.2" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/portal" "1.1.3" - "@chakra-ui/transition" "1.1.0" - "@chakra-ui/utils" "1.4.0" +"@chakra-ui/menu@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@chakra-ui/menu/-/menu-1.6.1.tgz#bb53a3dd4c67a06a8afdd41650ef345070fc3f3e" + integrity sha512-B2zFB4h1gcNmmwa/gBYaGKZC8jrSfWfANzVha59/ffiPPit62ss1dx7VptlluaHVBXzgTHutgP3PU1v/Arxtng== + dependencies: + "@chakra-ui/clickable" "1.1.5" + "@chakra-ui/descendant" "2.0.1" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/popper" "2.1.2" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/transition" "1.3.0" + "@chakra-ui/utils" "1.8.0" + +"@chakra-ui/modal@1.8.5": + version "1.8.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/modal/-/modal-1.8.5.tgz#eeae999867985004b43545244392bb867514ab39" + integrity sha512-hccvOEHzLd7MxaCtc25bPZI8hA0Sm89mFC6mn3XA/Uxh64jFhnbM0sqsQqXmfNi3RlBsHnXa9+COrPfa2ZuHPQ== + dependencies: + "@chakra-ui/close-button" "1.1.9" + "@chakra-ui/focus-lock" "1.1.8" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/portal" "1.2.5" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/transition" "1.3.0" + "@chakra-ui/utils" "1.8.0" aria-hidden "^1.1.1" react-remove-scroll "2.4.1" -"@chakra-ui/number-input@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/number-input/-/number-input-1.1.3.tgz#6035270af9e9177cb64521a94f9ab76b067e4eb3" - integrity sha512-vISd0obMse4EWVkowIyTE4JUfhzrPCjQkzdv9PgLEk1b5F5TVcKMCDAwPlsHhQGciaKnXRaMcOCTVeRS8oUoUQ== +"@chakra-ui/number-input@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/number-input/-/number-input-1.2.5.tgz#3cf29d420ef6f22dd245cf6b385659411eafb6d3" + integrity sha512-ysK6DooI8B9GL0pDG6myIQwJQbIq+RDfmoLPMzpoiWDvxE5eZ2pXJ0bIz70pQAIzmjfxc5WB7E8b81L0jVVaZQ== dependencies: - "@chakra-ui/counter" "1.0.9" - "@chakra-ui/form-control" "1.2.3" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/icon" "1.1.3" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/counter" "1.1.5" + "@chakra-ui/form-control" "1.3.5" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/icon" "1.1.9" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/pin-input@1.4.2": - version "1.4.2" - resolved "https://registry.yarnpkg.com/@chakra-ui/pin-input/-/pin-input-1.4.2.tgz#01b461210eca8b7878b2a49b9cdf5c96a1f9082d" - integrity sha512-7aS2IP67hfV1agnji8FemBJl1M7zqu2X5DTIy2NxuTNqU3/H8douMwVnNUrjavSltve4OYycxY8gPK/h9u4/uQ== +"@chakra-ui/pin-input@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@chakra-ui/pin-input/-/pin-input-1.6.1.tgz#c86f0559779e79b789268c113f3f950237e2cffd" + integrity sha512-phgnyrbaYysuvv401eozqdxk5aw5bnlZnw9hFIAkA3NqngBuvAMuZlM9WziBUQ6URX7lLtrvtbLPTYrOdMEm1g== dependencies: - "@chakra-ui/descendant" "1.0.9" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/descendant" "2.0.1" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/popover@1.3.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/popover/-/popover-1.3.0.tgz#df32ede440f36c26fadc3dcc754955aa9ddbd462" - integrity sha512-iVDe1A8BU4kImXKdOnaHnA0swcMXkkBYMxuTyhXVqvJwb+8uZgZ1OUdvYOgSEk/dA+idMQX7pINE0snw0W7USg== +"@chakra-ui/popover@1.7.0": + version "1.7.0" + resolved "https://registry.yarnpkg.com/@chakra-ui/popover/-/popover-1.7.0.tgz#fe5c366c7b4d2caf8a208faa850a4414a656927c" + integrity sha512-zD4ZjzfT6ZJFiIdlA/INaxlmmOvmOlqHHDXoP9zq0stA/JrZoCAZTvdlWFlRM+V+xZbgs1PYwS2bk/I1E9SaBw== dependencies: - "@chakra-ui/close-button" "1.1.3" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/popper" "2.0.0" - "@chakra-ui/portal" "1.1.3" - "@chakra-ui/transition" "1.1.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/close-button" "1.1.9" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/popper" "2.1.2" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/popper@2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/popper/-/popper-2.0.0.tgz#48cf74a434cd4e292b9304b65ef8a391b149ddd9" - integrity sha512-7qm1Zms9YOhtx48Zo8TvZ5pw0Uz91gK7D+2B2Lu9W3hNovDUA0/UuHcSnDf9iBZAQqwNkyYakXtJsYPOEdhbkw== +"@chakra-ui/popper@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@chakra-ui/popper/-/popper-2.1.2.tgz#a3948c338c0ff4948a659fad34dcbc3d2bf7ea31" + integrity sha512-X0GlUwOlX4coDO+JOYR52HZt8HsWGZmwL8GWr+DmRZvjBfRl7zKv8cAd4urYz5zwAo9TQfMb5bitjM/HPl3MvA== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/react-utils" "1.1.2" "@popperjs/core" "2.4.4" -"@chakra-ui/portal@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/portal/-/portal-1.1.3.tgz#a74f4f98540c3b6c3fbcb143831292374664af14" - integrity sha512-8CvC7YCFcvJV+LyuEP6EDMEZKl+C0u4A6L6JeUbkGNMqY3QcWhg+7+v0jrnq4ZvpZdpBSlzqRYX+dvZTJkZYjg== +"@chakra-ui/portal@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/portal/-/portal-1.2.5.tgz#2d966acb1e51c2d6be4eb5dc8a8a0affffc99513" + integrity sha512-yF78YaQI5MISo5sBqCVc51RXFcf5LeY+0C4Yvy7MoKtuyIMVGI3SvFfsCjZoMSw55MNTtqMrI7u5kIQKP37rzA== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/progress@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/progress/-/progress-1.1.3.tgz#80d61c3dbbb35a1e692d209fb370583b584c2ddf" - integrity sha512-SLxLfDAFQ1kiIp8SKwTUPaCldiHKgLg8F9exOt0KDX0nUen/u1PLvpeFKh+rX850ahn+/3tpIUOg1gPq4S9USw== +"@chakra-ui/progress@1.1.9": + version "1.1.9" + resolved "https://registry.yarnpkg.com/@chakra-ui/progress/-/progress-1.1.9.tgz#382d9cec6b45516525bb3a5f5d44b096ee50e66e" + integrity sha512-htnBK3pS7ZYv3vBP+qvFWsjfK1+lJgEuj7SzlLxZKk4jF0gtuZ60STCEn1j3JVZg7gWHDGTJcylipSTzqoXP/w== dependencies: - "@chakra-ui/theme-tools" "1.1.1" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/theme-tools" "1.1.7" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/radio@1.2.5": - version "1.2.5" - resolved "https://registry.yarnpkg.com/@chakra-ui/radio/-/radio-1.2.5.tgz#0ef48110c702616cbd15949082b462e21029a7d7" - integrity sha512-GtcNlDaD3cWBCY8Yr6S5AZgryY/w2rxL4b+uli2l3X1dlh1313IyOGR4HHpBKDGa+vr2tGUJA9f9oO4bhfGLPg== +"@chakra-ui/radio@1.3.5": + version "1.3.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/radio/-/radio-1.3.5.tgz#740b3a52115efae05eee1e8ddf9843303e6916c9" + integrity sha512-dTkQYZ0hS1g+YjPNcn0OgBgRtKciQtDv317D167trFkwm9z5ngebUdcnr1HO2YjO7qFnq4KCnjW2ouvQHIuBmQ== dependencies: - "@chakra-ui/form-control" "1.2.3" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" - "@chakra-ui/visually-hidden" "1.0.6" + "@chakra-ui/form-control" "1.3.5" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" + "@chakra-ui/visually-hidden" "1.0.12" -"@chakra-ui/react@^1.3.4": - version "1.4.1" - resolved "https://registry.yarnpkg.com/@chakra-ui/react/-/react-1.4.1.tgz#95caaaea65373dce2d64ad9803dad56bfdcf0315" - integrity sha512-jbFWV++0yhTJF92CulRDgkNkoMe589fdrc3YUyAh5ZqadNVB3qJoJxaTZLFV8gl8EUMCOu0Fu+EQkSoqkpzKGw== - dependencies: - "@chakra-ui/accordion" "1.1.4" - "@chakra-ui/alert" "1.1.3" - "@chakra-ui/avatar" "1.1.4" - "@chakra-ui/breadcrumb" "1.1.3" - "@chakra-ui/button" "1.1.4" - "@chakra-ui/checkbox" "1.3.0" - "@chakra-ui/close-button" "1.1.3" - "@chakra-ui/control-box" "1.0.6" - "@chakra-ui/counter" "1.0.9" - "@chakra-ui/css-reset" "1.0.0" - "@chakra-ui/editable" "1.1.3" - "@chakra-ui/form-control" "1.2.3" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/icon" "1.1.3" - "@chakra-ui/image" "1.0.9" - "@chakra-ui/input" "1.1.4" - "@chakra-ui/layout" "1.3.3" - "@chakra-ui/live-region" "1.0.6" - "@chakra-ui/media-query" "1.0.7" - "@chakra-ui/menu" "1.2.0" - "@chakra-ui/modal" "1.7.0" - "@chakra-ui/number-input" "1.1.3" - "@chakra-ui/pin-input" "1.4.2" - "@chakra-ui/popover" "1.3.0" - "@chakra-ui/popper" "2.0.0" - "@chakra-ui/portal" "1.1.3" - "@chakra-ui/progress" "1.1.3" - "@chakra-ui/radio" "1.2.5" - "@chakra-ui/select" "1.1.3" - "@chakra-ui/skeleton" "1.1.6" - "@chakra-ui/slider" "1.1.3" - "@chakra-ui/spinner" "1.1.3" - "@chakra-ui/stat" "1.1.3" - "@chakra-ui/switch" "1.1.5" - "@chakra-ui/system" "1.5.1" - "@chakra-ui/table" "1.1.3" - "@chakra-ui/tabs" "1.2.1" - "@chakra-ui/tag" "1.1.3" - "@chakra-ui/textarea" "1.1.3" - "@chakra-ui/theme" "1.7.1" - "@chakra-ui/toast" "1.2.0" - "@chakra-ui/tooltip" "1.2.0" - "@chakra-ui/transition" "1.1.0" - "@chakra-ui/utils" "1.4.0" - "@chakra-ui/visually-hidden" "1.0.6" - -"@chakra-ui/select@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/select/-/select-1.1.3.tgz#9ca61436a2881960b576f4968ffedb7a77d499cb" - integrity sha512-t5x7rM29aSc0LHMIvXMRvyNhufG5t+MNhZZZyxCnV8ySN/8IfpA01umGGn7Pz1qmME3gUw1Jz1paMvik0WTFsA== +"@chakra-ui/react-env@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@chakra-ui/react-env/-/react-env-1.0.4.tgz#b053d44d61b63dcf5997b24bdf2454bdcdb1b1ed" + integrity sha512-FkZR/WW7WI6LR3Wgm+OeuDhTNCcD77U/yWAxSo1tvEqqL5uUvD6Bi/Ko9E3lI5yhdoJ4t99ksf1vUBlrMG92vw== dependencies: - "@chakra-ui/form-control" "1.2.3" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/skeleton@1.1.6": - version "1.1.6" - resolved "https://registry.yarnpkg.com/@chakra-ui/skeleton/-/skeleton-1.1.6.tgz#2a3cceb101574a7dfaad4489bb73714a17a0a1d2" - integrity sha512-wzm9fCaLhTTO0i0bjlEc2PrVRgZcOEpxvJth9sFHOwd1aYGDigMhmi3azOzfWMeVcAoAkHBOkwa4o1BosD69rQ== +"@chakra-ui/react-utils@1.1.2", "@chakra-ui/react-utils@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@chakra-ui/react-utils/-/react-utils-1.1.2.tgz#7ea80b6ae25bd7b182095cc9ffaad23c464408b5" + integrity sha512-S8jPVKGZH2qF7ZGxl/0DF/dXXI2AxDNGf4Ahi2LGHqajMvqBB7vtYIRRmIA7+jAnErhzO8WUi3i4Z7oScp6xSA== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/media-query" "1.0.7" - "@chakra-ui/system" "1.5.1" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "^1.7.0" -"@chakra-ui/slider@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/slider/-/slider-1.1.3.tgz#e55740fff98269d2dccd03b8b8542389cf0b0122" - integrity sha512-stEzjnJBd2r3XAKOPSnufHHKAHrsuU2BQelJUzCpGxJtFYFSGvfNLMgQj3LjuqaIDjbOxSU4mkJYZ/KS+kXwFA== +"@chakra-ui/react@^1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@chakra-ui/react/-/react-1.6.1.tgz#1cee3d11eaa7d482614db3a27e9b4fcf66fb45ba" + integrity sha512-gHRy6Cv7PIepND0qT+CKTR2hQRNS7MIQ7dhx1ecYQyYwcbYPiH8aY3o3YCkanp4eyFGCQEC/tdRP8PZZozCkdA== + dependencies: + "@chakra-ui/accordion" "1.3.1" + "@chakra-ui/alert" "1.2.5" + "@chakra-ui/avatar" "1.2.5" + "@chakra-ui/breadcrumb" "1.2.5" + "@chakra-ui/button" "1.3.2" + "@chakra-ui/checkbox" "1.5.2" + "@chakra-ui/close-button" "1.1.9" + "@chakra-ui/control-box" "1.0.12" + "@chakra-ui/counter" "1.1.5" + "@chakra-ui/css-reset" "1.0.0" + "@chakra-ui/editable" "1.2.5" + "@chakra-ui/form-control" "1.3.5" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/icon" "1.1.9" + "@chakra-ui/image" "1.0.15" + "@chakra-ui/input" "1.2.5" + "@chakra-ui/layout" "1.4.5" + "@chakra-ui/live-region" "1.0.12" + "@chakra-ui/media-query" "1.0.13" + "@chakra-ui/menu" "1.6.1" + "@chakra-ui/modal" "1.8.5" + "@chakra-ui/number-input" "1.2.5" + "@chakra-ui/pin-input" "1.6.1" + "@chakra-ui/popover" "1.7.0" + "@chakra-ui/popper" "2.1.2" + "@chakra-ui/portal" "1.2.5" + "@chakra-ui/progress" "1.1.9" + "@chakra-ui/radio" "1.3.5" + "@chakra-ui/react-env" "1.0.4" + "@chakra-ui/select" "1.1.9" + "@chakra-ui/skeleton" "1.1.12" + "@chakra-ui/slider" "1.2.5" + "@chakra-ui/spinner" "1.1.9" + "@chakra-ui/stat" "1.1.9" + "@chakra-ui/switch" "1.2.5" + "@chakra-ui/system" "1.6.5" + "@chakra-ui/table" "1.2.4" + "@chakra-ui/tabs" "1.5.1" + "@chakra-ui/tag" "1.1.9" + "@chakra-ui/textarea" "1.1.9" + "@chakra-ui/theme" "1.8.5" + "@chakra-ui/toast" "1.2.6" + "@chakra-ui/tooltip" "1.3.5" + "@chakra-ui/transition" "1.3.0" + "@chakra-ui/utils" "1.8.0" + "@chakra-ui/visually-hidden" "1.0.12" + +"@chakra-ui/select@1.1.9": + version "1.1.9" + resolved "https://registry.yarnpkg.com/@chakra-ui/select/-/select-1.1.9.tgz#5cf16d6d663e7b0c26c7667cc811debc890cd180" + integrity sha512-EgYGAwbRRECIlcl2M5F0kQoc6oJE/MjaI3e3OCUAxFvtdTrJ0+DJFUseuNUNfgf4l9IQZOWwsSniSoRXmYMwqg== + dependencies: + "@chakra-ui/form-control" "1.3.5" + "@chakra-ui/utils" "1.8.0" + +"@chakra-ui/skeleton@1.1.12": + version "1.1.12" + resolved "https://registry.yarnpkg.com/@chakra-ui/skeleton/-/skeleton-1.1.12.tgz#e7f48f1b52135de67e40d9f680fb7ca29c1fc199" + integrity sha512-0zmAAYP5XTrQk0rryxSYPtEbgUBdRfuqxaRONP06xmzCH2LCgzHKOpSaWrvShHjesdLI7SWJeC7jK2bLQ6xKog== + dependencies: + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/media-query" "1.0.13" + "@chakra-ui/system" "1.6.5" + "@chakra-ui/utils" "1.8.0" + +"@chakra-ui/slider@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/slider/-/slider-1.2.5.tgz#99bc1946036d8620ec59f3eecea2482fe6b38313" + integrity sha512-DLNWuwQ2n+OENZk98U70rwg6ZQ9/23ZW4j79yOMrBm7+StlVEfEDVwA8pJigLmgvq0D8mmvZSxZY4aZ/bkaguw== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/spinner@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/spinner/-/spinner-1.1.3.tgz#97e96acfe4fdb2b3e41d2ddd01e14ca923b1cb37" - integrity sha512-kJZ7PhJ3wQdwSvfcUHUf3w8xULO3xepjKQZ3GV+p/F+xxl1/i+kvBIvDqGpDNHK3Fke27DW9jfOUMVqFa3Jrpw== +"@chakra-ui/spinner@1.1.9": + version "1.1.9" + resolved "https://registry.yarnpkg.com/@chakra-ui/spinner/-/spinner-1.1.9.tgz#c66d9e2b988d8a6067ae6a47e59ddf59614ad04f" + integrity sha512-G0EEEkFaPCElGjetPmR5Uup4t3reUxKUUPQIBA9ZZzdYESGmdQG7PTDgJpo/p42IpPQqHBIG1fL+7y2IapYu0w== dependencies: - "@chakra-ui/utils" "1.4.0" - "@chakra-ui/visually-hidden" "1.0.6" + "@chakra-ui/utils" "1.8.0" + "@chakra-ui/visually-hidden" "1.0.12" -"@chakra-ui/stat@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/stat/-/stat-1.1.3.tgz#9d7418bb0240c7401b39dc14303cc97ff244745a" - integrity sha512-ewEuymCAIKs9g+gS5s+kXmxdggrP5rBaFEIcQ/EPEO4zXFkTbPPPf1ZbUECfq53qrGnNzYZSIx2hqGaW2k0c3w== +"@chakra-ui/stat@1.1.9": + version "1.1.9" + resolved "https://registry.yarnpkg.com/@chakra-ui/stat/-/stat-1.1.9.tgz#650bdb6575627c42e8aabcb4d54fa3c045ebb332" + integrity sha512-F/wXCyFNxkNY8irAvhE6qlkni3pkoxqLANLTGrLijINkVP48wHykAurAorfg5KH+3jrJjRxC4RPruWhGFzE+Cw== dependencies: - "@chakra-ui/icon" "1.1.3" - "@chakra-ui/utils" "1.4.0" - "@chakra-ui/visually-hidden" "1.0.6" + "@chakra-ui/icon" "1.1.9" + "@chakra-ui/utils" "1.8.0" + "@chakra-ui/visually-hidden" "1.0.12" -"@chakra-ui/styled-system@1.9.1": - version "1.9.1" - resolved "https://registry.yarnpkg.com/@chakra-ui/styled-system/-/styled-system-1.9.1.tgz#4541a3944c64fc2c2705685abe76faaf1591c7f3" - integrity sha512-LS/1oW2BP1/wY+GbBpoMTdb1ezVx1P32zR52OUn4ANogsDcB/7/DMMj6tuS4yaZEZ9AKliORI+p0CD9P/ETyQA== +"@chakra-ui/styled-system@1.10.5": + version "1.10.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/styled-system/-/styled-system-1.10.5.tgz#1e93770b748977b2142af2471374cbfa3b682f74" + integrity sha512-DR8jbZlFD6xB/zoyXx95gt0OUwtHUTXYl8onIt+W3m8l73KKsv87B3Z9ByJgJcztLbcAr/tlEp48wCfkIXdrDQ== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" csstype "^3.0.6" -"@chakra-ui/switch@1.1.5": - version "1.1.5" - resolved "https://registry.yarnpkg.com/@chakra-ui/switch/-/switch-1.1.5.tgz#9d7ae9f20047cdeafbd3ed01f8432e7e2e659b37" - integrity sha512-zezIVpoAIKaspZShnkMkveT6wa7N8VIhlZhcLl32zaBr5W865t8eGM0uFcNpGu6FU881bMR9dRvK/S7OXnzqTw== +"@chakra-ui/switch@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/switch/-/switch-1.2.5.tgz#90beab9d8af78417006982312027753b038805d1" + integrity sha512-JzAlrQ/F18TzhwVGebr3IS4DaWiEVZfQ5ARKaOYKdgDfxNk39L6seTOheQVCczjk6fF/0Gweso9ivTACXv2dKg== dependencies: - "@chakra-ui/checkbox" "1.3.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/checkbox" "1.5.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/system@1.5.1": - version "1.5.1" - resolved "https://registry.yarnpkg.com/@chakra-ui/system/-/system-1.5.1.tgz#ab668a5b5d8f8899cd18c14d59546dff7df4537f" - integrity sha512-rbuCWqWHYvZ4Y89WsSkMvTPFP8usz3JN0/a36nsLzQ0YLG0Ehi6jUhjDSTbm9pQXh8cUpojGGNFtKW79HyIoXQ== +"@chakra-ui/system@1.6.5": + version "1.6.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/system/-/system-1.6.5.tgz#9b44db4573e32af8d6ad03047a0a1c6aba407e68" + integrity sha512-mGRl+fhypY9ISFD8VrjVGXBPb2ds1TdpYbfJlRZLphZWVze6W2z1Y5geK67ybIx9pTzcChhsgIHeePzWb6wZaQ== dependencies: - "@chakra-ui/color-mode" "1.1.2" - "@chakra-ui/styled-system" "1.9.1" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/color-mode" "1.1.8" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/styled-system" "1.10.5" + "@chakra-ui/utils" "1.8.0" react-fast-compare "3.2.0" -"@chakra-ui/table@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/table/-/table-1.1.3.tgz#d7463c59afa3d702d5e93a22ebdf803d4da4601c" - integrity sha512-vP4AJ+rXtB9l3YnlCvhdfbGzrs18lq9Nq9fDRNf8p8u2vtuiSTVar/kHXUO/JBI0NCpUfdAkELTxH3LonO7z4w== +"@chakra-ui/table@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@chakra-ui/table/-/table-1.2.4.tgz#f2de9cea0fa0b37f9abb35222bf3ba5b7636bdb5" + integrity sha512-nB+kSy07d5v4WCsiUB86P0tfajL847VD3GDDauW+x0HlJRI0Q7UFOnJZGANM3VliA/LDci5PjIIDDAL3cW3ISg== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/tabs@1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@chakra-ui/tabs/-/tabs-1.2.1.tgz#606168281ff8fe9916ccf2f79fa454136d5d2ae0" - integrity sha512-+nzHORo+obsZ3582qp8585WdCc9tRGu9I21edsaPwaYy3HJJNYh3/qXIaqW9xKbMuQmdPjGJlPN+sLWlda0gbw== +"@chakra-ui/tabs@1.5.1": + version "1.5.1" + resolved "https://registry.yarnpkg.com/@chakra-ui/tabs/-/tabs-1.5.1.tgz#ab3a3d20cba422304a42c23f3b8b19679774f1d0" + integrity sha512-SHDEKvVkGSX1okx5qV8tTBA74wdBeQkX3UnQTEnKckrie9lG98i2kzd8SdXoKrq/+38duv0oer4Cc2erVhobyA== dependencies: - "@chakra-ui/clickable" "1.0.6" - "@chakra-ui/descendant" "1.0.9" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/clickable" "1.1.5" + "@chakra-ui/descendant" "2.0.1" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/tag@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/tag/-/tag-1.1.3.tgz#bacda9ad0890028a4f31247b380942049a9c1467" - integrity sha512-qc4AfYDobBp7JhRcbIzmJal5BruJCZlvbcS/Ee8RPHUFIj9ZuaYB3OiZzGyScJxEzNch6kTxOSl5hooj+UYvig== +"@chakra-ui/tag@1.1.9": + version "1.1.9" + resolved "https://registry.yarnpkg.com/@chakra-ui/tag/-/tag-1.1.9.tgz#65357d2645ead56760bf14cf463602e9188a6a53" + integrity sha512-njANMXFoBaOWW4pNUDwwcUk1ejJ4OcPesTmYOBGjbVDBrag22kiAN3bSn8DKplKBhE/nHyJuhR2v9ndfb0BpLg== dependencies: - "@chakra-ui/icon" "1.1.3" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/icon" "1.1.9" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/textarea@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@chakra-ui/textarea/-/textarea-1.1.3.tgz#fa029bdb8ac917d061f2ef5fd2c55c67868c62d1" - integrity sha512-/lWWBP3JBhiQMjdthKi4kRiMz7M22xZPshaWFYJJ/SNKZVdSExhQDQmG6ytUcSvI4nNRV8N5kFkZMP8MZ7j35g== +"@chakra-ui/textarea@1.1.9": + version "1.1.9" + resolved "https://registry.yarnpkg.com/@chakra-ui/textarea/-/textarea-1.1.9.tgz#6bc6a3f9fec0470b55370399188c1f9f72c151d5" + integrity sha512-4UyfyPC2EqTIlOwm//4RzdmvqR49yszYigiOLcDQ6pxBNQ0ciHTIDyJlRUEn/q6ti2OnSmArcDQZ7XgF5lPEyg== dependencies: - "@chakra-ui/form-control" "1.2.3" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/form-control" "1.3.5" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/theme-tools@1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@chakra-ui/theme-tools/-/theme-tools-1.1.1.tgz#c5e2fda87fb9602f637ea8360e4084503faa4f2c" - integrity sha512-ECW8IF0DxMxQgwgn4SPKPbHXhp1fnz50WtInNTvyXHvFZSst0ANn8k+WRAMb9qXolzuZw0sFjhd8U59rtIIpwA== +"@chakra-ui/theme-tools@1.1.7": + version "1.1.7" + resolved "https://registry.yarnpkg.com/@chakra-ui/theme-tools/-/theme-tools-1.1.7.tgz#ffd3dc2d22a935e1a1df47b155c4cef60ff3f603" + integrity sha512-i8KpxjgoQPsoWAyHX+4kRMSioN9WVfyky+2sdFDHhEuDNL/iNYfKQMZjt8RR67apK/dSswMqh5UF2lWSM+lQhw== dependencies: - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" "@types/tinycolor2" "1.4.2" tinycolor2 "1.4.2" -"@chakra-ui/theme@1.7.1": - version "1.7.1" - resolved "https://registry.yarnpkg.com/@chakra-ui/theme/-/theme-1.7.1.tgz#71491d9610cfe3558b56ec92c4037e2a5de9528a" - integrity sha512-O1bAt9mbVdSF8u2QGewtA//rr4Vldo6p7tm3zFXm4vRfY0gQdvx3oEDxBWZX/KFRZbWV8c2IPmP0/fy4LWM0Ig== +"@chakra-ui/theme@1.8.5": + version "1.8.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/theme/-/theme-1.8.5.tgz#83c916c7d14c029a8124765252fc36ae22946439" + integrity sha512-bFYJA4/kdqVmjEpxjh+30jejR3bJtUtTYfuLRtfdo7NkgY13H9OReP08nDdarhXq6rA/891lkYGWDXvWwXpaqw== dependencies: - "@chakra-ui/theme-tools" "1.1.1" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/theme-tools" "1.1.7" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/toast@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/toast/-/toast-1.2.0.tgz#ebe43c611b81bfb5ed67ba462630f75e96610f1b" - integrity sha512-KbCofqnZRmCGaOtdctI8Q3GUXy5nzpnv+hKCWApC0i3GQononcXU5b/FYre+MM+egp9nqtds9u8/NhC1CCn4xg== - dependencies: - "@chakra-ui/alert" "1.1.3" - "@chakra-ui/close-button" "1.1.3" - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/theme" "1.7.1" - "@chakra-ui/transition" "1.1.0" - "@chakra-ui/utils" "1.4.0" - "@reach/alert" "0.13.0" - -"@chakra-ui/tooltip@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/tooltip/-/tooltip-1.2.0.tgz#4d5d784a954b33c3be05e2277c0b1d97e6ab1ff3" - integrity sha512-u6PA8amBTmUNbeCs73+A/BEhJwvK7BJz9UlPA3IwAXup2upzQlRWqgb1fYYME+fl4S7UZNGmjExBlanLq4ZxSQ== +"@chakra-ui/toast@1.2.6": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@chakra-ui/toast/-/toast-1.2.6.tgz#e110e492f62a6dc9c14a5e4a2ec420dc0fca2960" + integrity sha512-ARQyRiRLmTUf0dNRNiIXwoXgPlSFbRYBrjmumxoyIJILRpBQP1SP4kSCgO/KtI6jMDGVuIFdVsJoTSgcZLMHyw== + dependencies: + "@chakra-ui/alert" "1.2.5" + "@chakra-ui/close-button" "1.1.9" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/theme" "1.8.5" + "@chakra-ui/transition" "1.3.0" + "@chakra-ui/utils" "1.8.0" + "@reach/alert" "0.13.2" + +"@chakra-ui/tooltip@1.3.5": + version "1.3.5" + resolved "https://registry.yarnpkg.com/@chakra-ui/tooltip/-/tooltip-1.3.5.tgz#9d986fbad2b7e7e7c0754d221340faff1e5e705a" + integrity sha512-3KjOmhgrk38UvxT1Ct+xPYtOPBcbNAEU6v1Yvyl2NcNlihTVekDTLnF8GY2eRtAzKuB6JuhfsS2R+y7ubARamw== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/popper" "2.0.0" - "@chakra-ui/portal" "1.1.3" - "@chakra-ui/utils" "1.4.0" - "@chakra-ui/visually-hidden" "1.0.6" + "@chakra-ui/hooks" "1.5.2" + "@chakra-ui/popper" "2.1.2" + "@chakra-ui/portal" "1.2.5" + "@chakra-ui/react-utils" "1.1.2" + "@chakra-ui/utils" "1.8.0" + "@chakra-ui/visually-hidden" "1.0.12" -"@chakra-ui/transition@1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/transition/-/transition-1.1.0.tgz#7ac05f7dd2e817f321099b54d654cb1ce82b9035" - integrity sha512-cCOTfxvRF8zNfXF9R4CYBiCx24fzWvpAVbQrkjA+A4oUSFYwuqIt72H9fnCOCAh/q84NNKVFuXqtUCawuFjxcg== +"@chakra-ui/transition@1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@chakra-ui/transition/-/transition-1.3.0.tgz#fcaea97b7b828eef483609d7ecc8900b9623317a" + integrity sha512-IahxR4lNhFxS51r55nwdMJtxlYqEqNz9fcCOHWG4X9mhtp89SP7q57paJfpPLMND+Y2QNM6nWhGICN+w5g6zkg== dependencies: - "@chakra-ui/hooks" "1.2.0" - "@chakra-ui/utils" "1.4.0" + "@chakra-ui/utils" "1.8.0" -"@chakra-ui/utils@1.4.0": - version "1.4.0" - resolved "https://registry.yarnpkg.com/@chakra-ui/utils/-/utils-1.4.0.tgz#4b20c522141d68dc62b5c7be929bb6d1596db190" - integrity sha512-ND7NIilpncnkzcNBnAySGBFpOXb3urO1X9esSY0+M5zXiq+VSSfK9JuoJd3ilvV4Dbwgzil7hMAELBuHme4uTg== +"@chakra-ui/utils@1.8.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@chakra-ui/utils/-/utils-1.8.0.tgz#f7aad8175cc5a26a1d2795dc78691bbc21fd539e" + integrity sha512-BWIhKcXnLbOIwCTKeNcOStNwk9RyYVA9xRRFPGK6Kp3EhrxP0rDwAbu4D3o3qAc+yhIDhGmPaIj1jRXHB5DTfg== dependencies: "@types/lodash.mergewith" "4.6.6" - "@types/object-assign" "4.0.30" css-box-model "1.2.1" + framesync "5.3.0" lodash.mergewith "4.6.2" -"@chakra-ui/visually-hidden@1.0.6": - version "1.0.6" - resolved "https://registry.yarnpkg.com/@chakra-ui/visually-hidden/-/visually-hidden-1.0.6.tgz#c47578afbbd9396996c4ee3d7ce05d91f00a43c1" - integrity sha512-PM1QOGabhMAQyfNUq05GKSUr85dJdQGeQZVO6Gw8QOJ29+J/xS6Qpyvvn358OrWVYQjw3eEQ15ALEGXb/HJfeA== +"@chakra-ui/utils@^1.7.0": + version "1.7.0" + resolved "https://registry.yarnpkg.com/@chakra-ui/utils/-/utils-1.7.0.tgz#21c4807f56487284b93773da22665b63730a784c" + integrity sha512-fMZOnWBh/TzAYufkpeVw3yMNxKmYHEkB93luDpIJgxGt4mrbl+A66pd5GvJSGK1N34WYbjWRiJrrSvRI548eZA== dependencies: - "@chakra-ui/utils" "1.4.0" + "@types/lodash.mergewith" "4.6.6" + css-box-model "1.2.1" + framesync "5.3.0" + lodash.mergewith "4.6.2" + +"@chakra-ui/visually-hidden@1.0.12": + version "1.0.12" + resolved "https://registry.yarnpkg.com/@chakra-ui/visually-hidden/-/visually-hidden-1.0.12.tgz#9a5cf5c65c2e933a891442c4dd66767cf2f5f75f" + integrity sha512-CSRH1ySzI3QYP5QFrLFWgpy40rpddGn4Z+BjsZ5pf+FGp/KUEpFKUx8nZyGfHlT+flMCj6Tpu7ftPyWCh1HERg== + dependencies: + "@chakra-ui/utils" "1.8.0" "@cnakazawa/watch@^1.0.3": version "1.0.4" @@ -1467,7 +1519,7 @@ source-map "^0.5.7" stylis "^4.0.3" -"@emotion/cache@^11.1.3": +"@emotion/cache@^11.0.0", "@emotion/cache@^11.1.3": version "11.1.3" resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.1.3.tgz#c7683a9484bcd38d5562f2b9947873cf66829afd" integrity sha512-n4OWinUPJVaP6fXxWZD9OUeQ0lY7DvtmtSuqtRWT0Ofo/sBLCVSgb4/Oa0Q5eFxcwablRKjUXqXtNZVyEwCAuA== @@ -1507,7 +1559,7 @@ resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.5.tgz#2c40f81449a4e554e9fc6396910ed4843ec2be50" integrity sha512-igX9a37DR2ZPGYtV6suZ6whr8pTFtyHL3K/oLUotxpSVO2ASaprmAe2Dkq7tBo7CRY7MMDrAa9nuQP9/YG8FxQ== -"@emotion/react@^11.1.5": +"@emotion/react@^11.1.1", "@emotion/react@^11.1.5": version "11.1.5" resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.1.5.tgz#15e78f9822894cdc296e6f4e0688bac8120dfe66" integrity sha512-xfnZ9NJEv9SU9K2sxXM06lzjK245xSeHRpUh67eARBm3PBHjjKIZlfWZ7UQvD0Obvw6ZKjlC79uHrlzFYpOB/Q== @@ -1910,31 +1962,32 @@ resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.4.4.tgz#11d5db19bd178936ec89cd84519c4de439574398" integrity sha512-1oO6+dN5kdIA3sKPZhRGJTfGVP4SWV6KqlMOwry4J3HfyD68sl/3KmG7DeYUzvN+RbhXDnv/D8vNNB8168tAMg== -"@reach/alert@0.13.0": - version "0.13.0" - resolved "https://registry.yarnpkg.com/@reach/alert/-/alert-0.13.0.tgz#1f67b389f49af61286ef03a84f5a57bd3503dadf" - integrity sha512-5lpgRnlQ0JHBsRTPfKjD9aFPDZuLcaxTgD5PXdSLb+1CU8WgNbcy+7qSjqnu1uzWS2pQenIEBViV5wGpt63ADw== +"@reach/alert@0.13.2": + version "0.13.2" + resolved "https://registry.yarnpkg.com/@reach/alert/-/alert-0.13.2.tgz#71c4a848d51341f1d6d9eaae060975391c224870" + integrity sha512-LDz83AXCrClyq/MWe+0vaZfHp1Ytqn+kgL5VxG7rirUvmluWaj/snxzfNPWn0Ma4K2YENmXXRC/iHt5X95SqIg== dependencies: - "@reach/utils" "0.13.0" - "@reach/visually-hidden" "0.13.0" + "@reach/utils" "0.13.2" + "@reach/visually-hidden" "0.13.2" prop-types "^15.7.2" - tslib "^2.0.0" + tslib "^2.1.0" -"@reach/utils@0.13.0": - version "0.13.0" - resolved "https://registry.yarnpkg.com/@reach/utils/-/utils-0.13.0.tgz#2da775a910d8894bb34e1e94fe95842674f71844" - integrity sha512-dypxuyA1Qy3LHxzzyS7jFGPgCCR04b8UEn+Tv/aj6y9V578dULQqkcCyobrdEa+OI8lxH7dFFHa+jH8M/noBrQ== +"@reach/utils@0.13.2": + version "0.13.2" + resolved "https://registry.yarnpkg.com/@reach/utils/-/utils-0.13.2.tgz#87e8fef8ebfe583fa48250238a1a3ed03189fcc8" + integrity sha512-3ir6cN60zvUrwjOJu7C6jec/samqAeyAB12ZADK+qjnmQPdzSYldrFWwDVV5H0WkhbYXR3uh+eImu13hCetNPQ== dependencies: "@types/warning" "^3.0.0" - tslib "^2.0.0" + tslib "^2.1.0" warning "^4.0.3" -"@reach/visually-hidden@0.13.0": - version "0.13.0" - resolved "https://registry.yarnpkg.com/@reach/visually-hidden/-/visually-hidden-0.13.0.tgz#cace36d9bb80ffb797374fcaea989391b881038f" - integrity sha512-LF11WL9/495Q3d86xNy0VO6ylPI6SqF2xZGg9jpZSXLbFKpQ5Bf0qC7DOJfSf+/yb9WgPgB4m+a48Fz8AO6oZA== +"@reach/visually-hidden@0.13.2": + version "0.13.2" + resolved "https://registry.yarnpkg.com/@reach/visually-hidden/-/visually-hidden-0.13.2.tgz#ee21de376a7e57e60dc92d95a671073796caa17e" + integrity sha512-sPZwNS0/duOuG0mYwE5DmgEAzW9VhgU3aIt1+mrfT/xiT9Cdncqke+kRBQgU708q/Ttm9tWsoHni03nn/SuPTQ== dependencies: - tslib "^2.0.0" + prop-types "^15.7.2" + tslib "^2.1.0" "@sinonjs/commons@^1.7.0": version "1.8.2" @@ -2131,11 +2184,6 @@ resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA== -"@types/object-assign@4.0.30": - version "4.0.30" - resolved "https://registry.yarnpkg.com/@types/object-assign/-/object-assign-4.0.30.tgz#8949371d5a99f4381ee0f1df0a9b7a187e07e652" - integrity sha1-iUk3HVqZ9Dge4PHfCpt6GH4H5lI= - "@types/parse-json@^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" @@ -2151,6 +2199,13 @@ resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.3.tgz#2ab0d5da2e5815f94b0b9d4b95d1e5f243ab2ca7" integrity sha512-KfRL3PuHmqQLOG+2tGpRO26Ctg+Cq1E01D2DMriKEATHgWLfeNDmq9e29Q9WIky0dQ3NPkd1mzYH8Lm936Z9qw== +"@types/react-dom@*": + version "17.0.3" + resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-17.0.3.tgz#7fdf37b8af9d6d40127137865bb3fff8871d7ee1" + integrity sha512-4NnJbCeWE+8YBzupn/YrJxZ8VnjcJq5iR1laqQ1vkpQgBiA7bwk0Rp24fxsdNinzJY2U+HHS4dJJDPdoMjdJ7w== + dependencies: + "@types/react" "*" + "@types/react-dom@^17.0.2": version "17.0.2" resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-17.0.2.tgz#35654cf6c49ae162d5bc90843d5437dc38008d43" @@ -2175,6 +2230,23 @@ "@types/history" "*" "@types/react" "*" +"@types/react-select@^4.0.15": + version "4.0.15" + resolved "https://registry.yarnpkg.com/@types/react-select/-/react-select-4.0.15.tgz#2e6a1cff22c4bbae6c95b8dbee5b5097c12eae54" + integrity sha512-GPyBFYGMVFCtF4eg9riodEco+s2mflR10Nd5csx69+bcdvX6Uo9H/jgrIqovBU9yxBppB9DS66OwD6xxgVqOYQ== + dependencies: + "@emotion/serialize" "^1.0.0" + "@types/react" "*" + "@types/react-dom" "*" + "@types/react-transition-group" "*" + +"@types/react-transition-group@*": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.1.tgz#e1a3cb278df7f47f17b5082b1b3da17170bd44b1" + integrity sha512-vIo69qKKcYoJ8wKCJjwSgCTM+z3chw3g18dkrDfVX665tMH7tmbDxEAnPdey4gTlwZz5QuHGzd+hul0OVZDqqQ== + dependencies: + "@types/react" "*" + "@types/react@*", "@types/react@^17.0.3": version "17.0.3" resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.3.tgz#ba6e215368501ac3826951eef2904574c262cc79" @@ -2354,6 +2426,13 @@ "@typescript-eslint/types" "4.17.0" eslint-visitor-keys "^2.0.0" +"@vvo/tzdb@^6.7.0": + version "6.7.0" + resolved "https://registry.yarnpkg.com/@vvo/tzdb/-/tzdb-6.7.0.tgz#8a553df436a51b8813f75000206b080ad1e4a432" + integrity sha512-QBdgMZ9+fmSB/lFHfcjYaE9i43nGvXldzGENAcjFsawGMDUlXI7yZG6JBtPKscB1XFvPQfZ5kBsPzujZQdOr0A== + dependencies: + luxon "1.26.0" + "@webassemblyjs/ast@1.9.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.9.0.tgz#bd850604b4042459a5a41cd7d338cbed695ed964" @@ -4185,6 +4264,14 @@ dom-converter@^0.2: dependencies: utila "~0.4" +dom-helpers@^5.0.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/dom-helpers/-/dom-helpers-5.2.1.tgz#d9400536b2bf8225ad98fe052e029451ac40e902" + integrity sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA== + dependencies: + "@babel/runtime" "^7.8.7" + csstype "^3.0.2" + dom-serializer@0: version "0.2.2" resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51" @@ -5122,6 +5209,13 @@ framesync@5.2.0: resolved "https://registry.yarnpkg.com/framesync/-/framesync-5.2.0.tgz#f14480654cd05a6af4c72c9890cad93556841643" integrity sha512-dcl92w5SHc0o6pRK3//VBVNvu6WkYkiXmHG6ZIXrVzmgh0aDYMDAaoA3p3LH71JIdN5qmhDcfONFA4Lmq22tNA== +framesync@5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/framesync/-/framesync-5.3.0.tgz#0ecfc955e8f5a6ddc8fdb0cc024070947e1a0d9b" + integrity sha512-oc5m68HDO/tuK2blj7ZcdEBRx3p1PjrgHazL8GYEpvULhrtGIFbQArN6cQS2QhW8mitffaB+VYzMjDqBxxQeoA== + dependencies: + tslib "^2.1.0" + fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" @@ -6928,6 +7022,11 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" +luxon@1.26.0: + version "1.26.0" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-1.26.0.tgz#d3692361fda51473948252061d0f8561df02b578" + integrity sha512-+V5QIQ5f6CDXQpWNICELwjwuHdqeJM1UenlZWx5ujcRMc9venvluCjFb4t5NYLhb6IhkbMVOxzVuOqkgMxee2A== + lz-string@^1.4.4: version "1.4.4" resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26" @@ -6989,6 +7088,11 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= +memoize-one@^5.0.0: + version "5.2.1" + resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e" + integrity sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q== + memory-fs@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552" @@ -8093,7 +8197,7 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" -prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2: +prop-types@^15.5.8, prop-types@^15.6.0, prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2: version "15.7.2" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.7.2.tgz#52c41e75b8c87e72b9d9360e0206b99dcbffa6c5" integrity sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ== @@ -8298,6 +8402,13 @@ react-icons@^4.2.0: resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.2.0.tgz#6dda80c8a8f338ff96a1851424d63083282630d0" integrity sha512-rmzEDFt+AVXRzD7zDE21gcxyBizD/3NqjbX6cmViAgdqfJ2UiLer8927/QhhrXQV7dEj/1EGuOTPp7JnLYVJKQ== +react-input-autosize@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/react-input-autosize/-/react-input-autosize-3.0.0.tgz#6b5898c790d4478d69420b55441fcc31d5c50a85" + integrity sha512-nL9uS7jEs/zu8sqwFE5MAPx6pPkNAriACQ2rGLlqmKr2sPGtN7TXTyDdQt4lbNXVx7Uzadb40x8qotIuru6Rhg== + dependencies: + prop-types "^15.5.8" + "react-is@^16.12.0 || ^17.0.0", react-is@^17.0.1: version "17.0.1" resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.1.tgz#5b3531bd76a645a4c9fb6e693ed36419e3301339" @@ -8370,6 +8481,19 @@ react-router@5.2.0: tiny-invariant "^1.0.2" tiny-warning "^1.0.0" +react-select@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/react-select/-/react-select-4.3.0.tgz#6bde634ae7a378b49f3833c85c126f533483fa2e" + integrity sha512-SBPD1a3TJqE9zoI/jfOLCAoLr/neluaeokjOixr3zZ1vHezkom8K0A9J4QG9IWDqIDE9K/Mv+0y1GjidC2PDtQ== + dependencies: + "@babel/runtime" "^7.12.0" + "@emotion/cache" "^11.0.0" + "@emotion/react" "^11.1.1" + memoize-one "^5.0.0" + prop-types "^15.6.0" + react-input-autosize "^3.0.0" + react-transition-group "^4.3.0" + react-shallow-renderer@^16.13.1: version "16.14.1" resolved "https://registry.yarnpkg.com/react-shallow-renderer/-/react-shallow-renderer-16.14.1.tgz#bf0d02df8a519a558fd9b8215442efa5c840e124" @@ -8397,6 +8521,16 @@ react-test-renderer@^17.0.1: react-shallow-renderer "^16.13.1" scheduler "^0.20.1" +react-transition-group@^4.3.0: + version "4.4.1" + resolved "https://registry.yarnpkg.com/react-transition-group/-/react-transition-group-4.4.1.tgz#63868f9325a38ea5ee9535d828327f85773345c9" + integrity sha512-Djqr7OQ2aPUiYurhPalTrVy9ddmFCCzwhqQmtN+J3+3DzLO209Fdr70QrN8Z3DsglWql6iY1lDWAfpFiBtuKGw== + dependencies: + "@babel/runtime" "^7.5.5" + dom-helpers "^5.0.1" + loose-envify "^1.4.0" + prop-types "^15.6.2" + react@^16: version "16.14.0" resolved "https://registry.yarnpkg.com/react/-/react-16.14.0.tgz#94d776ddd0aaa37da3eda8fc5b6b18a4c9a3114d" @@ -9690,11 +9824,16 @@ tslib@^1.0.0, tslib@^1.10.0, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== -tslib@^2.0.0, tslib@^2.0.3: +tslib@^2.0.3: version "2.1.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a" integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A== +tslib@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.2.0.tgz#fb2c475977e35e241311ede2693cee1ec6698f5c" + integrity sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w== + tsutils@^3.17.1: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" diff --git a/airflow/utils/dag_processing.py b/airflow/utils/dag_processing.py index aa6b1b6097173..4b852342b91df 100644 --- a/airflow/utils/dag_processing.py +++ b/airflow/utils/dag_processing.py @@ -525,7 +525,7 @@ def __init__( os.set_blocking(self._signal_conn.fileno(), False) self._parallelism = conf.getint('scheduler', 'parsing_processes') - if 'sqlite' in conf.get('core', 'sql_alchemy_conn') and self._parallelism > 1: + if conf.get('core', 'sql_alchemy_conn').startswith('sqlite') and self._parallelism > 1: self.log.warning( "Because we cannot use more than 1 thread (parsing_processes = " "%d ) when using sqlite. So we set parallelism to 1.", @@ -719,7 +719,7 @@ def _run_parsing_loop(self): # "almost never happen" since the DagParsingStat object is # small, and in async mode this stat is not actually _required_ # for normal operation (It only drives "max runs") - self.log.debug("BlockingIOError recived trying to send DagParsingStat, ignoring") + self.log.debug("BlockingIOError received trying to send DagParsingStat, ignoring") if max_runs_reached: self.log.info( diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 79e9c9f8e5a61..979e020252702 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -613,7 +613,10 @@ def check_migrations(timeout): if source_heads == db_heads: break if ticker >= timeout: - raise TimeoutError(f"There are still unapplied migrations after {ticker} seconds.") + raise TimeoutError( + f"There are still unapplied migrations after {ticker} seconds. " + f"Migration Head(s) in DB: {db_heads} | Migration Head(s) in Source Code: {source_heads}" + ) ticker += 1 time.sleep(1) log.info('Waiting for migrations... %s second(s)', ticker) diff --git a/airflow/utils/decorators.py b/airflow/utils/decorators.py index 40fb9e0702f7b..60f033cf72d4a 100644 --- a/airflow/utils/decorators.py +++ b/airflow/utils/decorators.py @@ -17,87 +17,36 @@ # under the License. # -import inspect -import os -from copy import copy +import warnings from functools import wraps -from typing import Any, Callable, Dict, TypeVar, cast - -from airflow.exceptions import AirflowException - -signature = inspect.signature +from typing import Callable, TypeVar, cast T = TypeVar('T', bound=Callable) # pylint: disable=invalid-name def apply_defaults(func: T) -> T: """ - Function decorator that Looks for an argument named "default_args", and - fills the unspecified arguments from it. + This decorator is deprecated. - Since python2.* isn't clear about which arguments are missing when - calling a function, and that this can be quite confusing with multi-level - inheritance and argument defaults, this decorator also alerts with - specific information about the missing arguments. - """ - # Cache inspect.signature for the wrapper closure to avoid calling it - # at every decorated invocation. This is separate sig_cache created - # per decoration, i.e. each function decorated using apply_defaults will - # have a different sig_cache. - sig_cache = signature(func) - non_optional_args = { - name - for (name, param) in sig_cache.parameters.items() - if param.default == param.empty - and param.name != 'self' - and param.kind not in (param.VAR_POSITIONAL, param.VAR_KEYWORD) - } + In previous versions, all subclasses of BaseOperator must use apply_default decorator for the" + `default_args` feature to work properly. + In current version, it is optional. The decorator is applied automatically using the metaclass. + """ + warnings.warn( + "This decorator is deprecated. \n" + "\n" + "In previous versions, all subclasses of BaseOperator must use apply_default decorator for the" + "`default_args` feature to work properly.\n" + "\n" + "In current version, it is optional. The decorator is applied automatically using the metaclass.\n", + DeprecationWarning, + stacklevel=3, + ) + + # Make it still be a wrapper to keep the previous behaviour of an extra stack frame @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: - from airflow.models.dag import DagContext - - if len(args) > 1: - raise AirflowException("Use keyword arguments when initializing operators") - dag_args: Dict[str, Any] = {} - dag_params: Dict[str, Any] = {} - - dag = kwargs.get('dag') or DagContext.get_current_dag() - if dag: - dag_args = copy(dag.default_args) or {} - dag_params = copy(dag.params) or {} - - params = kwargs.get('params', {}) or {} - dag_params.update(params) - - default_args = {} - if 'default_args' in kwargs: - default_args = kwargs['default_args'] - if 'params' in default_args: - dag_params.update(default_args['params']) - del default_args['params'] - - dag_args.update(default_args) - default_args = dag_args - - for arg in sig_cache.parameters: - if arg not in kwargs and arg in default_args: - kwargs[arg] = default_args[arg] - - missing_args = list(non_optional_args - set(kwargs)) - if missing_args: - msg = f"Argument {missing_args} is required" - raise AirflowException(msg) - - kwargs['params'] = dag_params - - result = func(*args, **kwargs) - return result + def wrapper(*args, **kwargs): + return func(*args, **kwargs) return cast(T, wrapper) - - -if 'BUILDING_AIRFLOW_DOCS' in os.environ: - # flake8: noqa: F811 - # Monkey patch hook to get good function headers while building docs - apply_defaults = lambda x: x diff --git a/airflow/utils/log/secrets_masker.py b/airflow/utils/log/secrets_masker.py new file mode 100644 index 0000000000000..42e0e553aa68c --- /dev/null +++ b/airflow/utils/log/secrets_masker.py @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Mask sensitive information from logs""" +import collections +import logging +import re +from typing import TYPE_CHECKING, Iterable, Optional, Set, TypeVar, Union + +try: + # 3.8+ + from functools import cached_property +except ImportError: + from cached_property import cached_property + +try: + # 3.9+ + from functools import cache +except ImportError: + from functools import lru_cache + + cache = lru_cache(maxsize=None) + + +if TYPE_CHECKING: + from airflow.typing_compat import RePatternType + + RedactableItem = TypeVar('RedctableItem') + +DEFAULT_SENSITIVE_FIELDS = frozenset( + { + 'password', + 'secret', + 'passwd', + 'authorization', + 'api_key', + 'apikey', + 'access_token', + } +) +"""Names of fields (Connection extra, Variable key name etc.) that are deemed sensitive""" + + +@cache +def get_sensitive_variables_fields(): + """Get comma-separated sensitive Variable Fields from airflow.cfg.""" + from airflow.configuration import conf + + sensitive_fields = DEFAULT_SENSITIVE_FIELDS.copy() + sensitive_variable_fields = conf.get('core', 'sensitive_var_conn_names') + if sensitive_variable_fields: + sensitive_fields |= frozenset({field.strip() for field in sensitive_variable_fields.split(',')}) + return sensitive_fields + + +def should_hide_value_for_key(name): + """Should the value for this given name (Variable name, or key in conn.extra_dejson) be hidden""" + from airflow import settings + + if name and settings.HIDE_SENSITIVE_VAR_CONN_FIELDS: + name = name.strip().lower() + return any(s in name for s in get_sensitive_variables_fields()) + return False + + +def mask_secret(secret: Union[str, dict, Iterable], name: str = None) -> None: + """ + Mask a secret from appearing in the task logs. + + If ``name`` is provided, then it will only be masked if the name matches + one of the configured "sensitive" names. + + If ``secret`` is a dict or a iterable (excluding str) then it will be + recursively walked and keys with sensitive names will be hidden. + """ + # Delay import + from airflow import settings + + # Filtering all log messages is not a free process, so we only do it when + # running tasks + if not settings.MASK_SECRETS_IN_LOGS or not secret: + return + + _secrets_masker().add_mask(secret, name) + + +def redact(value: "RedactableItem", name: str = None) -> "RedactableItem": + """Redact any secrets found in ``value``.""" + return _secrets_masker().redact(value, name) + + +@cache +def _secrets_masker() -> "SecretsMasker": + + for flt in logging.getLogger('airflow.task').filters: + if isinstance(flt, SecretsMasker): + return flt + raise RuntimeError("No SecretsMasker found!") + + +class SecretsMasker(logging.Filter): + """Redact secrets from logs""" + + replacer: Optional["RePatternType"] = None + patterns: Set[str] + + ALREADY_FILTERED_FLAG = "__SecretsMasker_filtered" + + def __init__(self): + super().__init__() + self.patterns = set() + + @cached_property + def _record_attrs_to_ignore(self) -> Iterable[str]: + # Doing log.info(..., extra={'foo': 2}) sets extra properties on + # record, i.e. record.foo. And we need to filter those too. Fun + # + # Create a record, and look at what attributes are on it, and ignore + # all the default ones! + + record = logging.getLogRecordFactory()( + # name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None, + "x", + logging.INFO, + __file__, + 1, + "", + tuple(), + exc_info=None, + func="funcname", + ) + return frozenset(record.__dict__).difference({'msg', 'args'}) + + def filter(self, record) -> bool: + if self.ALREADY_FILTERED_FLAG in record.__dict__: + # Filters are attached to multiple handlers and logs, keep a + # "private" flag that stops us needing to process it more than once + return True + + if self.replacer: + for k, v in record.__dict__.items(): + if k in self._record_attrs_to_ignore: + continue + record.__dict__[k] = self.redact(v) + if record.exc_info: + exc = record.exc_info[1] + # I'm not sure if this is a good idea! + exc.args = (self.redact(v) for v in exc.args) + record.__dict__[self.ALREADY_FILTERED_FLAG] = True + + return True + + def _redact_all(self, item: "RedactableItem") -> "RedactableItem": + if isinstance(item, dict): + return {dict_key: self._redact_all(subval) for dict_key, subval in item.items()} + elif isinstance(item, str): + return '***' + elif isinstance(item, (tuple, set)): + # Turn set in to tuple! + return tuple(self._redact_all(subval) for subval in item) + elif isinstance(item, Iterable): + return list(self._redact_all(subval) for subval in item) + else: + return item + + # pylint: disable=too-many-return-statements + def redact(self, item: "RedactableItem", name: str = None) -> "RedactableItem": + """ + Redact an any secrets found in ``item``, if it is a string. + + If ``name`` is given, and it's a "sensitive" name (see + :func:`should_hide_value_for_key`) then all string values in the item + is redacted. + + """ + if name and should_hide_value_for_key(name): + return self._redact_all(item) + + if isinstance(item, dict): + return {dict_key: self.redact(subval, dict_key) for dict_key, subval in item.items()} + elif isinstance(item, str): + if self.replacer: + # We can't replace specific values, but the key-based redacting + # can still happen, so we can't short-circuit, we need to walk + # the structure. + return self.replacer.sub('***', item) + return item + elif isinstance(item, (tuple, set)): + # Turn set in to tuple! + return tuple(self.redact(subval) for subval in item) + elif isinstance(item, Iterable): + return list(self.redact(subval) for subval in item) + else: + return item + + # pylint: enable=too-many-return-statements + + def add_mask(self, secret: Union[str, dict, Iterable], name: str = None): + """Add a new secret to be masked to this filter instance.""" + if isinstance(secret, dict): + for k, v in secret.items(): + self.add_mask(v, k) + elif isinstance(secret, str): + pattern = re.escape(secret) + if pattern not in self.patterns and (not name or should_hide_value_for_key(name)): + self.patterns.add(pattern) + self.replacer = re.compile('|'.join(self.patterns)) + elif isinstance(secret, collections.abc.Iterable): + for v in secret: + self.add_mask(v, name) diff --git a/airflow/utils/python_virtualenv.py b/airflow/utils/python_virtualenv.py index 7780b98239795..2da609690da1c 100644 --- a/airflow/utils/python_virtualenv.py +++ b/airflow/utils/python_virtualenv.py @@ -100,7 +100,11 @@ def prepare_virtualenv( return f'{venv_directory}/bin/python' -def write_python_script(jinja_context: dict, filename: str): +def write_python_script( + jinja_context: dict, + filename: str, + render_template_as_native_obj: bool = False, +): """ Renders the python script to a file to execute in the virtual environment. @@ -109,8 +113,15 @@ def write_python_script(jinja_context: dict, filename: str): :type jinja_context: dict :param filename: The name of the file to dump the rendered script to. :type filename: str + :param render_template_as_native_obj: If ``True``, rendered Jinja template would be converted + to a native Python object """ template_loader = jinja2.FileSystemLoader(searchpath=os.path.dirname(__file__)) - template_env = jinja2.Environment(loader=template_loader, undefined=jinja2.StrictUndefined) + if render_template_as_native_obj: + template_env = jinja2.nativetypes.NativeEnvironment( + loader=template_loader, undefined=jinja2.StrictUndefined + ) + else: + template_env = jinja2.Environment(loader=template_loader, undefined=jinja2.StrictUndefined) template = template_env.get_template('python_virtualenv_script.jinja2') template.stream(**jinja_context).dump(filename) diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py index 3245fac5000cc..798c723da8723 100644 --- a/airflow/utils/timezone.py +++ b/airflow/utils/timezone.py @@ -169,5 +169,6 @@ def parse(string: str, timezone=None) -> DateTime: Parse a time string and return an aware datetime :param string: time string + :param timezone: the timezone """ return pendulum.parse(string, tz=timezone or TIMEZONE, strict=False) # type: ignore diff --git a/airflow/www/extensions/init_views.py b/airflow/www/extensions/init_views.py index 14e532b7ca9be..b6bebfd2fe695 100644 --- a/airflow/www/extensions/init_views.py +++ b/airflow/www/extensions/init_views.py @@ -97,6 +97,11 @@ def init_appbuilder_views(app): appbuilder.add_view( views.XComModelView, permissions.RESOURCE_XCOM, category=permissions.RESOURCE_ADMIN_MENU ) + appbuilder.add_view( + views.DagDependenciesView, + permissions.RESOURCE_DAG_DEPENDENCIES, + category=permissions.RESOURCE_BROWSE_MENU, + ) # add_view_no_menu to change item position. # I added link in extensions.init_appbuilder_links.init_appbuilder_links appbuilder.add_view_no_menu(views.RedocView) diff --git a/airflow/www/package.json b/airflow/www/package.json index 02d4b9aa8fb40..2e70b59a27b36 100644 --- a/airflow/www/package.json +++ b/airflow/www/package.json @@ -43,7 +43,6 @@ "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.0.1", "file-loader": "^6.0.0", - "handlebars": "^4.7.6", "imports-loader": "^1.1.0", "mini-css-extract-plugin": "0.9.0", "moment-locales-webpack-plugin": "^1.2.0", diff --git a/airflow/www/security.py b/airflow/www/security.py index d1bf9bc5565db..8768587fdb769 100644 --- a/airflow/www/security.py +++ b/airflow/www/security.py @@ -69,6 +69,7 @@ class AirflowSecurityManager(SecurityManager, LoggingMixin): # pylint: disable= VIEWER_PERMISSIONS = [ (permissions.ACTION_CAN_READ, permissions.RESOURCE_AUDIT_LOG), (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG), + (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_DEPENDENCIES), (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_CODE), (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_RUN), (permissions.ACTION_CAN_READ, permissions.RESOURCE_IMPORT_ERROR), @@ -124,6 +125,7 @@ class AirflowSecurityManager(SecurityManager, LoggingMixin): # pylint: disable= (permissions.ACTION_CAN_READ, permissions.RESOURCE_POOL), (permissions.ACTION_CAN_EDIT, permissions.RESOURCE_POOL), (permissions.ACTION_CAN_DELETE, permissions.RESOURCE_POOL), + (permissions.ACTION_CAN_READ, permissions.RESOURCE_PROVIDER), (permissions.ACTION_CAN_CREATE, permissions.RESOURCE_VARIABLE), (permissions.ACTION_CAN_READ, permissions.RESOURCE_VARIABLE), (permissions.ACTION_CAN_EDIT, permissions.RESOURCE_VARIABLE), diff --git a/airflow/www/static/css/calendar.css b/airflow/www/static/css/calendar.css new file mode 100644 index 0000000000000..3d578ce80e068 --- /dev/null +++ b/airflow/www/static/css/calendar.css @@ -0,0 +1,52 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +text.title { + font-size: 16px; + font-weight: bold; +} + +text.year-label { + font-size: 16px; + font-weight: bold; +} + +text.day-label { + font-size: 12px; +} + +text.status-label { + font-size: 11px; +} + +path.month { + stroke: black; + stroke-width: 2px; +} + +rect.day { + stroke: #ccc; + stroke-width: 1px; + cursor: pointer; +} + +.legend-item__swatch { + background-color: #fff; + border: 1px solid #ccc; +} diff --git a/airflow/www/static/css/dags.css b/airflow/www/static/css/dags.css index d6da029668e71..6d25949951f10 100644 --- a/airflow/www/static/css/dags.css +++ b/airflow/www/static/css/dags.css @@ -31,6 +31,10 @@ .dags-table-body { margin: 0 1px; overflow-x: auto; + + /* This allows small dropdowns to overlap the table element */ + padding-bottom: 55px; + margin-bottom: -55px; } .dags-table-more { diff --git a/airflow/www/static/css/graph.css b/airflow/www/static/css/graph.css index 6259d454a070c..f4c7b942c5a77 100644 --- a/airflow/www/static/css/graph.css +++ b/airflow/www/static/css/graph.css @@ -165,3 +165,30 @@ g.node.removed rect { background-color: #f0f0f0; cursor: move; } + +.legend-item.dag { + float: left; + background-color: #e8f7e4; +} + +.legend-item.trigger { + float: left; + background-color: #ffefeb; +} + +.legend-item.sensor { + float: left; + background-color: #e6f1f2; +} + +g.node.dag rect { + fill: #e8f7e4; +} + +g.node.trigger rect { + fill: #ffefeb; +} + +g.node.sensor rect { + fill: #e6f1f2; +} diff --git a/airflow/www/static/js/calendar.js b/airflow/www/static/js/calendar.js new file mode 100644 index 0000000000000..cac968a5f3368 --- /dev/null +++ b/airflow/www/static/js/calendar.js @@ -0,0 +1,342 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* global calendarData, statesColors, document, window, $, d3, moment */ +import getMetaValue from './meta_value'; + +const dagId = getMetaValue('dag_id'); +const treeUrl = getMetaValue('tree_url'); + +function getTreeViewURL(d) { + return `${treeUrl + }?dag_id=${encodeURIComponent(dagId) + }&base_date=${encodeURIComponent(d.toISOString())}`; +} + +// date helpers +function formatDay(d) { + return ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'][d]; +} + +function toMoment(y, m, d) { + return moment.utc([y, m, d]); +} + +function weekOfMonth(y, m, d) { + const monthOffset = toMoment(y, m, 1).day(); + const dayOfMonth = toMoment(y, m, d).date(); + return Math.floor((dayOfMonth + monthOffset - 1) / 7); +} + +function weekOfYear(y, m) { + const yearOffset = toMoment(y, 0, 1).day(); + const dayOfYear = toMoment(y, m, 1).dayOfYear(); + return Math.floor((dayOfYear + yearOffset - 1) / 7); +} + +function daysInMonth(y, m) { + const lastDay = toMoment(y, m, 1).add(1, 'month').subtract(1, 'day'); + return lastDay.date(); +} + +function weeksInMonth(y, m) { + const firstDay = toMoment(y, m, 1); + const monthOffset = firstDay.day(); + return Math.floor((daysInMonth(y, m) + monthOffset) / 7) + 1; +} + +const dateFormat = 'YYYY-MM-DD'; + +document.addEventListener('DOMContentLoaded', () => { + $('span.status_square').tooltip({ html: true }); + + // JSON.parse is faster for large payloads than an object literal + const rootData = JSON.parse(calendarData); + + const dayTip = d3.tip() + .attr('class', 'tooltip d3-tip') + .html((toolTipHtml) => toolTipHtml); + + // draw the calendar + function draw() { + // display constants + const leftRightMargin = 32; + const titleHeight = 24; + const yearLabelWidth = 34; + const dayLabelWidth = 14; + const dayLabelPadding = 4; + const yearPadding = 20; + const cellSize = 16; + const yearHeight = cellSize * 7 + 2; + const maxWeeksInYear = 53; + const legendHeight = 30; + const legendSwatchesPadding = 4; + const legendSwtchesTextWidth = 44; + + // group dag run stats by year -> month -> day -> state + let dagStates = d3 + .nest() + .key((dr) => moment.utc(dr.date, dateFormat).year()) + .key((dr) => moment.utc(dr.date, dateFormat).month()) + .key((dr) => moment.utc(dr.date, dateFormat).date()) + .key((dr) => dr.state) + .map(rootData.dag_states); + + // Make sure we have one year displayed for each year between the start and end dates. + // This also ensures we do not have show an empty calendar view when no dag runs exist. + const startYear = moment.utc(rootData.start_date, dateFormat).year(); + const endYear = moment.utc(rootData.end_date, dateFormat).year(); + for (let y = startYear; y <= endYear; y += 1) { + dagStates[y] = dagStates[y] || {}; + } + + dagStates = d3 + .entries(dagStates) + .map((keyVal) => ({ + year: keyVal.key, + dagStates: keyVal.value, + })) + .sort((data) => data.year); + + // root SVG element + const fullWidth = ( + leftRightMargin * 2 + yearLabelWidth + dayLabelWidth + + maxWeeksInYear * cellSize + ); + const yearsHeight = (yearHeight + yearPadding) * dagStates.length + yearPadding; + const fullHeight = titleHeight + legendHeight + yearsHeight; + + const svg = d3 + .select('#calendar-svg') + .attr('width', fullWidth) + .attr('height', fullHeight) + .call(dayTip); + + // Add the title + svg + .append('g') + .append('text') + .attr('x', fullWidth / 2) + .attr('y', 20) + .attr('text-anchor', 'middle') + .attr('class', 'title') + .text('DAG states'); + + // Add the legend + const legend = svg + .append('g') + .attr('transform', `translate(0, ${titleHeight + legendHeight / 2})`); + + let legendXOffset = fullWidth - leftRightMargin; + + function drawLegend(rightState, leftState, numSwatches = 1, swatchesWidth = cellSize) { + const startColor = statesColors[leftState || rightState]; + const endColor = statesColors[rightState]; + + legendXOffset -= legendSwtchesTextWidth; + legend + .append('text') + .attr('x', legendXOffset) + .attr('y', cellSize / 2) + .attr('text-anchor', 'start') + .attr('class', 'status-label') + .attr('alignment-baseline', 'middle') + .text(rightState); + legendXOffset -= legendSwatchesPadding; + + legendXOffset -= swatchesWidth; + legend + .append('g') + .attr('transform', `translate(${legendXOffset}, 0)`) + .selectAll('g') + .data(d3.range(numSwatches)) + .enter() + .append('rect') + .attr('x', (v) => v * (swatchesWidth / numSwatches)) + .attr('width', swatchesWidth / numSwatches) + .attr('height', cellSize) + .attr('class', 'day') + .attr('fill', (v) => d3.interpolateHsl(startColor, endColor)(v / numSwatches)); + legendXOffset -= legendSwatchesPadding; + + if (leftState !== undefined) { + legend + .append('text') + .attr('x', legendXOffset) + .attr('y', cellSize / 2) + .attr('text-anchor', 'end') + .attr('class', 'status-label') + .attr('alignment-baseline', 'middle') + .text(leftState); + legendXOffset -= legendSwtchesTextWidth; + } + } + + drawLegend('no_status'); + drawLegend('running'); + drawLegend('failed', 'success', 10, 100); + + // Add the years groups, each holding one year of data. + const years = svg + .append('g') + .attr('transform', `translate(${leftRightMargin}, ${titleHeight + legendHeight})`); + + const year = years + .selectAll('g') + .data(dagStates) + .enter() + .append('g') + .attr('transform', (d, i) => `translate(0, ${yearPadding + (yearHeight + yearPadding) * i})`); + + year + .append('text') + .attr('x', -yearHeight * 0.5) + .attr('transform', 'rotate(270)') + .attr('text-anchor', 'middle') + .attr('class', 'year-label') + .text((d) => d.year); + + // write day names + year + .append('g') + .attr('transform', `translate(${yearLabelWidth}, ${dayLabelPadding})`) + .attr('text-anchor', 'end') + .selectAll('g') + .data(d3.range(7)) + .enter() + .append('text') + .attr('y', (i) => (i + 0.5) * cellSize) + .attr('class', 'day-label') + .text(formatDay); + + // create months groups to old the individual day cells & month outline for each month. + const months = year + .append('g') + .attr('transform', `translate(${yearLabelWidth + dayLabelWidth}, 0)`); + + const month = months + .append('g') + .selectAll('g') + .data((data) => d3 + .range(12) + .map((i) => ({ + year: data.year, + month: i, + dagStates: data.dagStates[i] || {}, + }))) + .enter() + .append('g') + .attr('transform', (data) => `translate(${weekOfYear(data.year, data.month) * cellSize}, 0)`); + + const tipHtml = (data) => { + const stateCounts = d3.entries(data.dagStates).map((kv) => `${kv.value[0].count} ${kv.key}`); + const date = toMoment(data.year, data.month, data.day); + const daySr = formatDay(date.day()); + const dateStr = date.format(dateFormat); + return `${daySr} ${dateStr}
${stateCounts.join('
')}`; + }; + + // Create the day cells + month + .selectAll('g') + .data((data) => d3 + .range(daysInMonth(data.year, data.month)) + .map((i) => { + const day = i + 1; + const dagRunsByState = data.dagStates[day] || {}; + return { + year: data.year, + month: data.month, + day, + dagStates: dagRunsByState, + }; + })) + .enter() + .append('rect') + .attr('x', (data) => weekOfMonth(data.year, data.month, data.day) * cellSize) + .attr('y', (data) => toMoment(data.year, data.month, data.day).day() * cellSize) + .attr('width', cellSize) + .attr('height', cellSize) + .attr('class', 'day') + .attr('fill', (data) => { + const runningCount = (data.dagStates.running || [{ count: 0 }])[0].count; + if (runningCount > 0) return statesColors.running; + + const successCount = (data.dagStates.success || [{ count: 0 }])[0].count; + const failedCount = (data.dagStates.failed || [{ count: 0 }])[0].count; + if (successCount + failedCount === 0) return statesColors.no_status; + + let ratioFailures; + if (failedCount === 0) ratioFailures = 0; + else { + // We use a minimum color interpolation floor, so that days with low failures ratios + // don't appear almost as green as days with not failure at all. + const floor = 0.5; + ratioFailures = floor + (failedCount / (failedCount + successCount)) * (1 - floor); + } + return d3.interpolateHsl(statesColors.success, statesColors.failed)(ratioFailures); + }) + .on('click', (data) => { + window.location.href = getTreeViewURL( + // add 1 day and subtract 1 ms to not show any run from the next day. + toMoment(data.year, data.month, data.day).add(1, 'day').subtract(1, 'ms'), + ); + }) + .on('mouseover', function showTip(data) { + const tt = tipHtml(data); + dayTip.direction('n'); + dayTip.show(tt, this); + }) + .on('mouseout', function hideTip(data) { + dayTip.hide(data, this); + }); + + // add outline (path) around month + month + .selectAll('g') + .data((data) => [data]) + .enter() + .append('path') + .attr('class', 'month') + .style('fill', 'none') + .attr('d', (data) => { + const firstDayOffset = toMoment(data.year, data.month, 1).day(); + const lastDayOffset = toMoment(data.year, data.month, 1).add(1, 'month').day(); + const weeks = weeksInMonth(data.year, data.month); + return d3.svg.line()([ + [0, firstDayOffset * cellSize], + [cellSize, firstDayOffset * cellSize], + [cellSize, 0], + [weeks * cellSize, 0], + [weeks * cellSize, lastDayOffset * cellSize], + [(weeks - 1) * cellSize, lastDayOffset * cellSize], + [(weeks - 1) * cellSize, 7 * cellSize], + [0, 7 * cellSize], + [0, firstDayOffset * cellSize], + ]); + }); + } + + function update() { + $('#loading').remove(); + draw(); + } + + update(); +}); diff --git a/airflow/www/static/js/dag_dependencies.js b/airflow/www/static/js/dag_dependencies.js new file mode 100644 index 0000000000000..786d33707025d --- /dev/null +++ b/airflow/www/static/js/dag_dependencies.js @@ -0,0 +1,195 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + global d3, localStorage, dagreD3, nodes, edges, arrange, document, +*/ + +const highlightColor = '#000000'; +const upstreamColor = '#2020A0'; +const downstreamColor = '#0000FF'; +const initialStrokeWidth = '3px'; +const highlightStrokeWidth = '5px'; +const duration = 500; + +// Preparation of DagreD3 data structures +const g = new dagreD3.graphlib.Graph() + .setGraph({ + nodesep: 15, + ranksep: 15, + rankdir: arrange, + }) + .setDefaultEdgeLabel(() => ({ lineInterpolate: 'basis' })); + +// Set all nodes and styles +nodes.forEach((node) => { + g.setNode(node.id, node.value); +}); + +// Set edges +edges.forEach((edge) => { + g.setEdge(edge.u, edge.v); +}); + +const render = dagreD3.render(); +const svg = d3.select('#graph-svg'); +const innerSvg = d3.select('#graph-svg g'); + +innerSvg.call(render, g); + +// Returns true if a node's id or its children's id matches search_text +function nodeMatches(nodeId, searchText) { + if (nodeId.indexOf(searchText) > -1) return true; + return false; +} + +function highlightNodes(nodes, color, strokeWidth) { + nodes.forEach((nodeid) => { + const myNode = g.node(nodeid).elem; + d3.select(myNode) + .selectAll('rect,circle') + .style('stroke', color) + .style('stroke-width', strokeWidth); + }); +} + +let zoom = null; + +function setUpZoomSupport() { + // Set up zoom support for Graph + zoom = d3.behavior.zoom().on('zoom', () => { + innerSvg.attr('transform', `translate(${d3.event.translate})scale(${d3.event.scale})`); + }); + svg.call(zoom); + + // Centering the DAG on load + // Get Dagre Graph dimensions + const graphWidth = g.graph().width; + const graphHeight = g.graph().height; + // Get SVG dimensions + const padding = 20; + const svgBb = svg.node().getBoundingClientRect(); + const width = svgBb.width - padding * 2; + const height = svgBb.height - padding; // we are not centering the dag vertically + + // Calculate applicable scale for zoom + const zoomScale = Math.min( + Math.min(width / graphWidth, height / graphHeight), + 1.5, // cap zoom level to 1.5 so nodes are not too large + ); + + zoom.translate([(width / 2) - ((graphWidth * zoomScale) / 2) + padding, padding]); + zoom.scale(zoomScale); + zoom.event(innerSvg); +} + +function setUpNodeHighlighting(focusItem = null) { + d3.selectAll('g.node').on('mouseover', function (d) { + d3.select(this).selectAll('rect').style('stroke', highlightColor); + highlightNodes(g.predecessors(d), upstreamColor, highlightStrokeWidth); + highlightNodes(g.successors(d), downstreamColor, highlightStrokeWidth); + const adjacentNodeNames = [d, ...g.predecessors(d), ...g.successors(d)]; + d3.selectAll('g.nodes g.node') + .filter((x) => !adjacentNodeNames.includes(x)) + .style('opacity', 0.2); + const adjacentEdges = g.nodeEdges(d); + d3.selectAll('g.edgePath')[0] + // eslint-disable-next-line no-underscore-dangle + .filter((x) => !adjacentEdges.includes(x.__data__)) + .forEach((x) => { + d3.select(x).style('opacity', 0.2); + }); + }); + + d3.selectAll('g.node').on('mouseout', function (d) { + d3.select(this).selectAll('rect,circle').style('stroke', null); + highlightNodes(g.predecessors(d), null, initialStrokeWidth); + highlightNodes(g.successors(d), null, initialStrokeWidth); + d3.selectAll('g.node') + .style('opacity', 1); + d3.selectAll('g.node rect') + .style('stroke-width', initialStrokeWidth); + d3.selectAll('g.edgePath') + .style('opacity', 1); + if (focusItem) { + localStorage.removeItem(focusItem); + } + }); +} + +function searchboxHighlighting(s) { + let match = null; + + d3.selectAll('g.nodes g.node').forEach(function forEach(d) { + if (s === '') { + d3.select('g.edgePaths') + .transition().duration(duration) + .style('opacity', 1); + d3.select(this) + .transition().duration(duration) + .style('opacity', 1) + .selectAll('rect') + .style('stroke-width', initialStrokeWidth); + } else { + d3.select('g.edgePaths') + .transition().duration(duration) + .style('opacity', 0.2); + if (nodeMatches(d, s)) { + if (!match) match = this; + d3.select(this) + .transition().duration(duration) + .style('opacity', 1) + .selectAll('rect') + .style('stroke-width', highlightStrokeWidth); + } else { + d3.select(this) + .transition() + .style('opacity', 0.2).duration(duration) + .selectAll('rect') + .style('stroke-width', initialStrokeWidth); + } + } + }); + + // This moves the matched node to the center of the graph area + if (match) { + const transform = d3.transform(d3.select(match).attr('transform')); + + const svgBb = svg.node().getBoundingClientRect(); + transform.translate = [ + svgBb.width / 2 - transform.translate[0], + svgBb.height / 2 - transform.translate[1], + ]; + transform.scale = [1, 1]; + + if (zoom !== null) { + zoom.translate(transform.translate); + zoom.scale(1); + zoom.event(innerSvg); + } + } +} + +setUpNodeHighlighting(); +setUpZoomSupport(); + +d3.select('#searchbox').on('keyup', () => { + const s = document.getElementById('searchbox').value; + searchboxHighlighting(s); +}); diff --git a/airflow/www/templates/airflow/calendar.html b/airflow/www/templates/airflow/calendar.html new file mode 100644 index 0000000000000..4caf2d9b3bf55 --- /dev/null +++ b/airflow/www/templates/airflow/calendar.html @@ -0,0 +1,53 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +{% extends "airflow/dag.html" %} +{% block page_title %}{{ dag.dag_id }} - Tree - {{ appbuilder.app_name }}{% endblock %} + +{% block head_css %} + {{ super() }} + + + +{% endblock %} + +{% block content %} + {{ super() }} +
+
+ + +
+{% endblock %} + +{% block tail_js %} + {{ super() }} + + + + +{% endblock %} diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 20d4043f933b6..4dac17c343205 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -92,6 +92,10 @@

  • Graph View
    • +
  • + + Calendar View +
  • Task Duration
    • diff --git a/airflow/www/templates/airflow/dag_dependencies.html b/airflow/www/templates/airflow/dag_dependencies.html new file mode 100644 index 0000000000000..708b7fc263953 --- /dev/null +++ b/airflow/www/templates/airflow/dag_dependencies.html @@ -0,0 +1,75 @@ +{# +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +#} + +{% extends "airflow/main.html" %} + +{% block title %}Airflow - DAG Dependencies{% endblock %} + +{% block head_css %} +{{ super() }} + +{% endblock %} + +{% block content %} +{{ super() }} +

    + {{ title }} +
    + +
    +

    +
    +
    +
    + dag + trigger + sensor +
    +
    Last refresh: {{ last_refresh }}
    +
    + +
    +
    +
    + + + + + + +
    +
    +{% endblock %} + +{% block tail %} + {{ super() }} + + + + + + +{% endblock %} diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html index 051bf79fc1e8e..4b0fd95bd7ca5 100644 --- a/airflow/www/templates/airflow/dags.html +++ b/airflow/www/templates/airflow/dags.html @@ -233,6 +233,10 @@

    {{ page_title }}

    Duration + + + Calendar + Graph diff --git a/airflow/www/utils.py b/airflow/www/utils.py index af34536c4f84f..b0c93ba404ba8 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -28,7 +28,6 @@ from pygments import highlight, lexers from pygments.formatters import HtmlFormatter # noqa pylint: disable=no-name-in-module -from airflow.configuration import conf from airflow.utils import timezone from airflow.utils.code_utils import get_python_source from airflow.utils.json import AirflowJsonEncoder @@ -36,35 +35,33 @@ from airflow.www.forms import DateTimeWithTimezoneField from airflow.www.widgets import AirflowDateTimePickerWidget -DEFAULT_SENSITIVE_VARIABLE_FIELDS = [ - 'password', - 'secret', - 'passwd', - 'authorization', - 'api_key', - 'apikey', - 'access_token', -] +def get_sensitive_variables_fields(): # noqa: D103 + import warnings -def get_sensitive_variables_fields(): - """Get comma-separated sensitive Variable Fields from airflow.cfg.""" - sensitive_fields = set(DEFAULT_SENSITIVE_VARIABLE_FIELDS) - sensitive_variable_fields = conf.get('admin', 'sensitive_variable_fields') - if sensitive_variable_fields: - sensitive_fields.update({field.strip() for field in sensitive_variable_fields.split(',')}) - return sensitive_fields + from airflow.utils.log.secrets_masker import get_sensitive_variables_fields + + warnings.warn( + "This function is deprecated. Please use " + "`airflow.utils.log.secrets_masker.get_sensitive_variables_fields`", + DeprecationWarning, + stacklevel=2, + ) + return get_sensitive_variables_fields() -def should_hide_value_for_key(key_name): - """Returns True if hide_sensitive_variable_fields is True, else False""" - # It is possible via importing variables from file that a key is empty. - if key_name: - config_set = conf.getboolean('admin', 'hide_sensitive_variable_fields') +def should_hide_value_for_key(key_name): # noqa: D103 + import warnings - field_comp = any(s in key_name.strip().lower() for s in get_sensitive_variables_fields()) - return config_set and field_comp - return False + from airflow.utils.log.secrets_masker import should_hide_value_for_key + + warnings.warn( + "This function is deprecated. Please use " + "`airflow.utils.log.secrets_masker.should_hide_value_for_key`", + DeprecationWarning, + stacklevel=2, + ) + return should_hide_value_for_key(key_name) def get_params(**kwargs): @@ -464,7 +461,7 @@ def is_utcdatetime(self, col_name): filter_converter_class = UtcAwareFilterConverter -# This class is used directly (i.e. we cant tell Fab to use a different +# This class is used directly (i.e. we can't tell Fab to use a different # subclass) so we have no other option than to edit the conversion table in # place FieldConverter.conversion_table = ( diff --git a/airflow/www/views.py b/airflow/www/views.py index 8607b17b5ccdf..00e42e2dc63c7 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -99,6 +99,7 @@ from airflow.models.baseoperator import BaseOperator from airflow.models.dagcode import DagCode from airflow.models.dagrun import DagRun, DagRunType +from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskinstance import TaskInstance from airflow.providers_manager import ProvidersManager from airflow.security import permissions @@ -108,6 +109,7 @@ from airflow.utils.dates import infer_time_unit, scale_time_units from airflow.utils.docs import get_docs_url from airflow.utils.helpers import alchemy_to_dict +from airflow.utils.log import secrets_masker from airflow.utils.log.log_reader import TaskLogReader from airflow.utils.session import create_session, provide_session from airflow.utils.state import State @@ -2080,6 +2082,70 @@ def tree(self): external_log_name=external_log_name, ) + @expose('/calendar') + @auth.has_access( + [ + (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG), + (permissions.ACTION_CAN_READ, permissions.RESOURCE_TASK_INSTANCE), + ] + ) + @gzipped # pylint: disable=too-many-locals + @action_logging # pylint: disable=too-many-locals + def calendar(self): + """Get DAG runs as calendar""" + dag_id = request.args.get('dag_id') + dag = current_app.dag_bag.get_dag(dag_id) + if not dag: + flash(f'DAG "{dag_id}" seems to be missing from DagBag.', "error") + return redirect(url_for('Airflow.index')) + + root = request.args.get('root') + if root: + dag = dag.sub_dag(task_ids_or_regex=root, include_downstream=False, include_upstream=True) + + with create_session() as session: + dag_states = ( + session.query( + func.date(DagRun.execution_date).label('date'), + DagRun.state, + func.count('*').label('count'), + ) + .filter(DagRun.dag_id == dag.dag_id) + .group_by(func.date(DagRun.execution_date), DagRun.state) + .order_by(func.date(DagRun.execution_date).asc()) + .all() + ) + + dag_states = [ + { + # DATE() in SQLite and MySQL behave differently: + # SQLite returns a string, MySQL returns a date. + 'date': dr.date if isinstance(dr.date, str) else dr.date.isoformat(), + 'state': dr.state, + 'count': dr.count, + } + for dr in dag_states + ] + + data = { + 'dag_states': dag_states, + 'start_date': (dag.start_date or DateTime.utcnow()).date().isoformat(), + 'end_date': (dag.end_date or DateTime.utcnow()).date().isoformat(), + } + + doc_md = wwwutils.wrapped_markdown(getattr(dag, 'doc_md', None)) + + # avoid spaces to reduce payload size + data = htmlsafe_json_dumps(data, separators=(',', ':')) + + return self.render_template( + 'airflow/calendar.html', + dag=dag, + doc_md=doc_md, + data=data, + root=root, + ) + @expose('/graph') @auth.has_access( [ @@ -3281,7 +3347,7 @@ def hidden_field_formatter(self): """Formats hidden fields""" key = self.get('key') # noqa pylint: disable=no-member val = self.get('val') # noqa pylint: disable=no-member - if wwwutils.should_hide_value_for_key(key): + if secrets_masker.should_hide_value_for_key(key): return Markup('*' * 8) if val: return val @@ -3295,7 +3361,7 @@ def hidden_field_formatter(self): validators_columns = {'key': [validators.DataRequired()]} def prefill_form(self, form, request_id): # pylint: disable=unused-argument - if wwwutils.should_hide_value_for_key(form.key.data): + if secrets_masker.should_hide_value_for_key(form.key.data): form.val.data = '*' * 8 @action('muldelete', 'Delete', 'Are you sure you want to delete selected records?', single=False) @@ -3959,6 +4025,78 @@ def autocomplete(self, session=None): return wwwutils.json_response(payload) +class DagDependenciesView(AirflowBaseView): + """View to show dependencies between DAGs""" + + refresh_interval = timedelta( + seconds=conf.getint( + "webserver", + "dag_dependencies_refresh_interval", + fallback=conf.getint("scheduler", "dag_dir_list_interval"), + ) + ) + last_refresh = timezone.utcnow() - refresh_interval + nodes = [] + edges = [] + + @expose('/dag-dependencies') + @auth.has_access( + [ + (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_DEPENDENCIES), + ] + ) + @gzipped + @action_logging + def list(self): + """Display DAG dependencies""" + title = "DAG Dependencies" + + if timezone.utcnow() > self.last_refresh + self.refresh_interval: + if SerializedDagModel.get_max_last_updated_datetime() > self.last_refresh: + self._calculate_graph() + self.last_refresh = timezone.utcnow() + + return self.render_template( + "airflow/dag_dependencies.html", + title=title, + nodes=self.nodes, + edges=self.edges, + last_refresh=self.last_refresh.strftime("%Y-%m-%d %H:%M:%S"), + arrange=conf.get("webserver", "dag_orientation"), + width=request.args.get("width", "100%"), + height=request.args.get("height", "800"), + ) + + def _calculate_graph(self): + + nodes = [] + edges = [] + + for dag, dependencies in SerializedDagModel.get_dag_dependencies().items(): + dag_node_id = f"dag:{dag}" + nodes.append(self._node_dict(dag_node_id, dag, "dag")) + + for dep in dependencies: + + nodes.append(self._node_dict(dep.node_id, dep.dependency_id, dep.dependency_type)) + edges.extend( + [ + {"u": f"dag:{dep.source}", "v": dep.node_id}, + {"u": dep.node_id, "v": f"dag:{dep.target}"}, + ] + ) + + self.nodes = nodes + self.edges = edges + + @staticmethod + def _node_dict(node_id, label, node_class): + return { + "id": node_id, + "value": {"label": label, "rx": 5, "ry": 5, "class": node_class}, + } + + class CustomPermissionModelView(PermissionModelView): """Customize permission names for FAB's builtin PermissionModelView.""" diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index fb7e63cc953c3..d432f9138a40a 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -40,6 +40,7 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, dagCode: `${JS_DIR}/dag_code.js`, + dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], flash: `${CSS_DIR}/flash.css`, gantt: [`${CSS_DIR}/gantt.css`, `${JS_DIR}/gantt.js`], @@ -54,6 +55,7 @@ const config = { taskInstance: `${JS_DIR}/task_instance.js`, tiLog: `${JS_DIR}/ti_log.js`, tree: [`${CSS_DIR}/tree.css`, `${JS_DIR}/tree.js`], + calendar: [`${CSS_DIR}/calendar.css`, `${JS_DIR}/calendar.js`], circles: `${JS_DIR}/circles.js`, durationChart: `${JS_DIR}/duration_chart.js`, trigger: `${JS_DIR}/trigger.js`, diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index 1b07d0321e15f..fdb367e93e3b5 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -1656,7 +1656,7 @@ colorette@^1.2.0: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b" integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw== -commander@2, commander@^2.20.0, commander@~2.20.3: +commander@2, commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== @@ -3506,18 +3506,6 @@ graphlib@^2.1.8: dependencies: lodash "^4.17.15" -handlebars@^4.7.6: - version "4.7.6" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.6.tgz#d4c05c1baf90e9945f77aa68a7a219aa4a7df74e" - integrity sha512-1f2BACcBfiwAfStCKZNrUCgqNZkGsAT7UM3kkYtXuLo0KnaVfjKOyf7PRzB6++aK9STyT1Pd2ZCPe3EGOXleXA== - dependencies: - minimist "^1.2.5" - neo-async "^2.6.0" - source-map "^0.6.1" - wordwrap "^1.0.0" - optionalDependencies: - uglify-js "^3.1.4" - hard-rejection@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/hard-rejection/-/hard-rejection-2.1.0.tgz#1c6eda5c1685c63942766d79bb40ae773cecd883" @@ -4929,11 +4917,6 @@ neo-async@^2.5.0, neo-async@^2.6.1: resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== -neo-async@^2.6.0: - version "2.6.1" - resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.1.tgz#ac27ada66167fa8849a6addd837f6b189ad2081c" - integrity sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw== - nice-try@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" @@ -7479,14 +7462,6 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -uglify-js@^3.1.4: - version "3.7.6" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.7.6.tgz#0783daa867d4bc962a37cc92f67f6e3238c47485" - integrity sha512-yYqjArOYSxvqeeiYH2VGjZOqq6SVmhxzaPjJC1W2F9e+bqvFL9QXQ2osQuKUFjM2hGjKG2YclQnRKWQSt/nOTQ== - dependencies: - commander "~2.20.3" - source-map "~0.6.1" - unherit@^1.0.4: version "1.1.3" resolved "https://registry.yarnpkg.com/unherit/-/unherit-1.1.3.tgz#6c9b503f2b41b262330c80e91c8614abdaa69c22" @@ -7833,11 +7808,6 @@ word-wrap@^1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== -wordwrap@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" - integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus= - worker-farm@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8" diff --git a/breeze b/breeze index 2a8d4a1c45bcf..55a92c95bb532 100755 --- a/breeze +++ b/breeze @@ -141,9 +141,6 @@ function breeze::setup_default_breeze_constants() { # Which means that you do not have to start from scratch export PRESERVE_VOLUMES="false" - # If set to true, RBAC UI will not be used for 1.10 version - export DISABLE_RBAC="false" - # Sources by default are installed from local sources when using breeze AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM:="."} export AIRFLOW_SOURCES_FROM @@ -639,6 +636,7 @@ export AIRFLOW_IMAGE="${airflow_image}" export SQLITE_URL="${SQLITE_URL}" export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}" export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}" +export EXECUTOR="${EXECUTOR}" docker-compose --log-level INFO ${command} EOF chmod u+x "${file}" @@ -845,6 +843,12 @@ function breeze::parse_arguments() { echo shift 2 ;; + --executor) + export EXECUTOR="${2}" + echo "Using ${EXECUTOR} in cluster" + echo + shift 2 + ;; --helm-version) export HELM_VERSION="${2}" echo "Helm version: ${HELM_VERSION}" @@ -1216,12 +1220,6 @@ function breeze::parse_arguments() { echo shift ;; - --no-rbac-ui) - export DISABLE_RBAC="true" - echo "When installing Airflow 1.10, RBAC UI will be disabled." - echo - shift - ;; --use-packages-from-dist) export USE_PACKAGES_FROM_DIST="true" echo "Install packages found in dist folder when entering breeze." @@ -1568,6 +1566,10 @@ function breeze::prepare_formatted_versions() { fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") readonly FORMATTED_HELM_VERSIONS + FORMATTED_EXECUTORS=$(echo "${_breeze_allowed_executors=""}" | tr '\n' ' ' | + fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") + readonly FORMATTED_EXECUTORS + FORMATTED_KIND_OPERATIONS=$(echo "${_breeze_allowed_kind_operations=""}" | tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") readonly FORMATTED_KIND_OPERATIONS @@ -1843,7 +1845,7 @@ ${CMDNAME} generate-constraints [FLAGS] Generates pinned constraint files with all extras from setup.py. Those files are generated in files folder - separate files for different python version. Those constraint files when - pushed to orphan constraints-master, constraints-2-0 and constraints-1-10 branches are used + pushed to orphan constraints-master, constraints-2-0 branches are used to generate repeatable CI builds as well as run repeatable production image builds and upgrades when you want to include installing or updating some of the released providers released at the time particular airflow version was released. You can use those @@ -1877,14 +1879,23 @@ ${CMDNAME} prepare-airflow-packages [FLAGS] Prepares airflow packages (sdist and wheel) in dist folder. Note that prepare-provider-packages command cleans up the dist folder, so if you want also to generate provider packages, make sure you run prepare-provider-packages first, - and prepare-airflow-packages second. + and prepare-airflow-packages second. You can specify optional + --version-suffix-for-svn flag to generate rc candidate packages to upload to SVN or + --version-suffix-for-pypi flag to generate rc candidates for PyPI packages. You can also + provide both suffixes in case you prepare alpha/beta versions. The packages are prepared in + dist folder - General form: + Examples: - '${CMDNAME} prepare-airflow-packages + '${CMDNAME} prepare-airflow-packages --package-format wheel' or + '${CMDNAME} prepare-airflow-packages --version-suffix-for-svn rc1' or + '${CMDNAME} prepare-airflow-packages --version-suffix-for-pypi rc1' + '${CMDNAME} prepare-airflow-packages --version-suffix-for-pypi a1 + --version-suffix-for-svn a1' Flags: $(breeze::flag_packages) +$(breeze::flag_version_suffix) $(breeze::flag_verbosity) " readonly DETAILED_USAGE_PREPARE_AIRFLOW_PACKAGES @@ -2239,9 +2250,6 @@ function breeze::flag_airflow_variants() { -p, --python PYTHON_MAJOR_MINOR_VERSION Python version used for the image. This is always major/minor version. - Note that versions 2.7 and 3.5 are only valid when installing Airflow 1.10 with - --install-airflow-version or --install-airflow-reference flags. - One of: ${FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS} @@ -2340,6 +2348,7 @@ ${FORMATTED_INTEGRATIONS} # FORMATTED_KUBERNETES_VERSIONS # FORMATTED_KIND_VERSIONS # FORMATTED_HELM_VERSIONS +# FORMATTED_EXECUTORS # # # Outputs: @@ -2381,6 +2390,14 @@ ${FORMATTED_HELM_VERSIONS} Default: ${_breeze_default_helm_version:=} +--executor EXECUTOR + Executor to use in a kubernetes cluster. + One of: + +${FORMATTED_EXECUTORS} + + Default: ${_breeze_default_executor:=} + " } @@ -2414,13 +2431,13 @@ function breeze::flag_local_file_mounting() { function breeze::flag_build_different_airflow_version() { echo " -a, --install-airflow-version INSTALL_AIRFLOW_VERSION - Uses differen version of Airflow when building PROD image. + Uses different version of Airflow when building PROD image. ${FORMATTED_INSTALL_AIRFLOW_VERSIONS} -t, --install-airflow-reference INSTALL_AIRFLOW_REFERENCE Installs Airflow directly from reference in GitHub when building PROD image. - This can be a GitHub branch like master or v1-10-test, or a tag like 2.0.0a1. + This can be a GitHub branch like master or v2-0-test, or a tag like 2.0.0a1. --installation-method INSTALLATION_METHOD Method of installing Airflow in PROD image - either from the sources ('.') @@ -2465,10 +2482,6 @@ ${FORMATTED_USE_AIRFLOW_VERSION} it will install the packages after entering the image. This is useful for testing provider packages. ---no-rbac-ui - Disables RBAC UI when Airflow 1.10.* is installed. - - " } @@ -3021,6 +3034,7 @@ function breeze::print_star_line() { # KUBERNETES_VERSION # KIND_VERSION # HELM_VERSION +# EXECUTOR # POSTGRES_VERSION # MYSQL_VERSION # DOCKERHUB_USER @@ -3044,6 +3058,9 @@ function breeze::read_saved_environment_variables() { HELM_VERSION="${HELM_VERSION:=$(parameters::read_from_file HELM_VERSION)}" HELM_VERSION=${HELM_VERSION:=${_breeze_default_helm_version}} + EXECUTOR="${EXECUTOR:=$(parameters::read_from_file EXECUTOR)}" + EXECUTOR=${EXECUTOR:-${_breeze_default_executor}} + POSTGRES_VERSION="${POSTGRES_VERSION:=$(parameters::read_from_file POSTGRES_VERSION)}" POSTGRES_VERSION=${POSTGRES_VERSION:=${_breeze_default_postgres_version}} @@ -3066,7 +3083,7 @@ function breeze::read_saved_environment_variables() { ####################################################################################################### # # Checks if variables are correctly set and if they are - saves them so that they can be used across -# sessions. In case we are installing Airflow 1.10, the constants are set to match 1.10 line. +# sessions. # # In case the variables are matching expected values they are saved in ".build/VARIABLE_NAME" for # later reuse. If not, error is printed and the saved file is cleaned, so that next time @@ -3080,6 +3097,7 @@ function breeze::read_saved_environment_variables() { # KUBERNETES_VERSION # KIND_VERSION # HELM_VERSION +# EXECUTOR # POSTGRES_VERSION # MYSQL_VERSION # DOCKERHUB_USER @@ -3094,37 +3112,21 @@ function breeze::check_and_save_all_params() { parameters::check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python" if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then - if [[ ${INSTALL_AIRFLOW_REFERENCE=} == *1_10* ]]; then - export BRANCH_NAME="v1-10-test" - elif [[ ${INSTALL_AIRFLOW_REFERENCE=} == *2_0* ]]; then + if [[ ${INSTALL_AIRFLOW_REFERENCE=} == *2_0* ]]; then export BRANCH_NAME="v2-0-test" fi elif [[ -n "${INSTALL_AIRFLOW_VERSION=}" ]]; then - if [[ ${INSTALL_AIRFLOW_VERSION=} == *1.10* ]]; then - export BRANCH_NAME="v1-10-test" - elif [[ ${INSTALL_AIRFLOW_VERSION=} == *2.0* ]]; then + if [[ ${INSTALL_AIRFLOW_VERSION=} == *2.0* ]]; then export BRANCH_NAME="v2-0-test" fi fi - if [[ ${PYTHON_MAJOR_MINOR_VERSION} == "2.7" || ${PYTHON_MAJOR_MINOR_VERSION} == "3.5" ]]; then - if [[ ${BRANCH_NAME} == "master" || ${BRANCH_NAME} == "v2-0-test" ]]; then - echo - echo "${COLOR_RED}ERROR: The ${PYTHON_MAJOR_MINOR_VERSION} can only be used when installing Airflow 1.10.* ${COLOR_RESET}" - echo - echo "You can use it only when you specify 1.10 Airflow via --install-airflow-version" - echo "or --install-airflow-reference and they point to 1.10 version of Airflow" - echo - exit 1 - fi - fi - - parameters::check_and_save_allowed_param "BACKEND" "backend" "--backend" parameters::check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode" parameters::check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version" parameters::check_and_save_allowed_param "KIND_VERSION" "KinD version" "--kind-version" parameters::check_and_save_allowed_param "HELM_VERSION" "Helm version" "--helm-version" + parameters::check_and_save_allowed_param "EXECUTOR" "Executors" "--executor" parameters::check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version" parameters::check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version" parameters::check_and_save_allowed_param "GITHUB_REGISTRY" "GitHub Registry" "--github-registry" @@ -3530,7 +3532,7 @@ function breeze::run_breeze_command() { ;; perform_prepare_airflow_packages) docker_engine_resources::check_all_resources - build_airflow_packages::build_airflow_packages + runs::run_prepare_airflow_packages ;; perform_prepare_provider_packages) docker_engine_resources::check_all_resources diff --git a/breeze-complete b/breeze-complete index 14a1dd35ba8c4..18724b33c2fe7 100644 --- a/breeze-complete +++ b/breeze-complete @@ -23,7 +23,7 @@ # by the BATS tests automatically during pre-commit and CI # Those cannot be made read-only as the breeze-complete must be re-sourceable -_breeze_allowed_python_major_minor_versions="2.7 3.5 3.6 3.7 3.8" +_breeze_allowed_python_major_minor_versions="3.6 3.7 3.8" _breeze_allowed_backends="sqlite mysql postgres" _breeze_allowed_integrations="cassandra kerberos mongo openldap pinot rabbitmq redis statsd trino all" _breeze_allowed_generate_constraints_modes="source-providers pypi-providers no-providers" @@ -36,6 +36,7 @@ _breeze_allowed_kind_versions="v0.10.0" _breeze_allowed_mysql_versions="5.7 8" _breeze_allowed_postgres_versions="9.6 10 11 12 13" _breeze_allowed_kind_operations="start stop restart status deploy test shell k9s" +_breeze_allowed_executors="KubernetesExecutor CeleryExecutor LocalExecutor CeleryKubernetesExecutor" _breeze_allowed_test_types="All Core Providers API CLI Integration Other WWW Postgres MySQL Helm Quarantined" _breeze_allowed_package_formats="both sdist wheel" _breeze_allowed_installation_methods=". apache-airflow" @@ -50,6 +51,7 @@ _breeze_allowed_installation_methods=". apache-airflow" _breeze_default_kubernetes_version=$(echo "${_breeze_allowed_kubernetes_versions}" | awk '{print $1}') _breeze_default_helm_version=$(echo "${_breeze_allowed_helm_versions}" | awk '{print $1}') _breeze_default_kind_version=$(echo "${_breeze_allowed_kind_versions}" | awk '{print $1}') + _breeze_default_executor=$(echo "${_breeze_allowed_executors}" | awk '{print $1}') _breeze_default_postgres_version=$(echo "${_breeze_allowed_postgres_versions}" | awk '{print $1}') _breeze_default_mysql_version=$(echo "${_breeze_allowed_mysql_versions}" | awk '{print $1}') _breeze_default_test_type=$(echo "${_breeze_allowed_test_types}" | awk '{print $1}') @@ -60,8 +62,6 @@ _breeze_allowed_install_airflow_versions=$(cat <<-EOF 2.0.2 2.0.1 2.0.0 -1.10.15 -1.10.14 wheel sdist EOF @@ -185,6 +185,7 @@ use-packages-from-dist no-rbac-ui package-format: upgrade-to-newer-dependencies use-airflow-version: cleanup-docker-context-files test-type: preserve-volumes dry-run-docker +executor: " _breeze_commands=" @@ -305,6 +306,9 @@ function breeze_complete::get_known_values_breeze() { --test-type) _breeze_known_values="${_breeze_allowed_test_types}" ;; + --executor) + _breeze_known_values="${_breeze_allowed_executors}" + ;; --package-format) _breeze_known_values="${_breeze_allowed_package_formats}" ;; diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 02f8b0fff8167..556a10cc7d6ab 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -20,7 +20,7 @@ kind: Pod metadata: name: dummy-name spec: -{{- if .Values.dags.gitSync.enabled }} +{{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} initContainers: {{- include "git_sync_container" (dict "Values" .Values "is_init" "true") | indent 4 }} {{- end }} @@ -40,7 +40,7 @@ spec: ports: [] volumeMounts: - mountPath: {{ template "airflow_logs" . }} - name: airflow-logs + name: logs - name: config mountPath: {{ template "airflow_config_path" . }} subPath: airflow.cfg @@ -61,17 +61,11 @@ spec: name: git-sync-ssh-key subPath: ssh {{- end }} -{{- if .Values.dags.persistence.enabled }} +{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }} - mountPath: {{ include "airflow_dags_mount_path" . }} name: dags readOnly: true {{- end }} -{{- if .Values.dags.gitSync.enabled }} - - mountPath: {{ include "airflow_dags" . }} - name: dags - readOnly: true - subPath: {{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subPath }} -{{- end }} {{- if .Values.workers.extraVolumeMounts }} {{ toYaml .Values.workers.extraVolumeMounts | indent 8 }} {{- end }} @@ -100,8 +94,14 @@ spec: {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }} {{- include "git_sync_ssh_key_volume" . | indent 2 }} {{- end }} +{{- if .Values.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ template "airflow_logs_volume_claim" . }} +{{- else }} - emptyDir: {} - name: airflow-logs + name: logs +{{- end }} {{- if .Values.dags.gitSync.knownHosts }} - configMap: defaultMode: 288 diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 2f1641b011b9c..7e9b7df9d5ec6 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -300,8 +300,8 @@ If release name contains chart name it will be used as a full name. {{- end }} {{ define "pgbouncer_config" }} -{{- $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s.svc.cluster.local" .Release.Name "postgresql" .Release.Namespace) }} -{{- $pgResultBackendHost := .Values.data.resultBackendConnection.host | default (printf "%s-%s.%s.svc.cluster.local" .Release.Name "postgresql" .Release.Namespace) }} +{{- $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} +{{- $pgResultBackendHost := .Values.data.resultBackendConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} [databases] {{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ .Values.data.metadataConnection.db }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }} {{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ .Values.data.resultBackendConnection.db }} port={{ .Values.data.resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }} diff --git a/chart/templates/configmaps/configmap.yaml b/chart/templates/configmaps/configmap.yaml index ebabf412efd0d..120ca85c94a4d 100644 --- a/chart/templates/configmaps/configmap.yaml +++ b/chart/templates/configmaps/configmap.yaml @@ -51,12 +51,14 @@ data: {{ .Values.dags.gitSync.knownHosts | nindent 4 }} {{- end }} {{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }} +{{- if semverCompare ">=1.10.12" .Values.airflowVersion }} pod_template_file.yaml: |- {{- if .Values.podTemplate }} {{ .Values.podTemplate | nindent 4 }} {{- else }} {{ tpl (.Files.Get "files/pod-template-file.kubernetes-helm-yaml") . | nindent 4 }} {{- end }} +{{- end }} {{- if .Values.kerberos.enabled }} krb5.conf: | {{ tpl .Values.kerberos.config . | nindent 4 }} diff --git a/chart/templates/flower/flower-deployment.yaml b/chart/templates/flower/flower-deployment.yaml index 5910c25514f00..2019f3f2b63c1 100644 --- a/chart/templates/flower/flower-deployment.yaml +++ b/chart/templates/flower/flower-deployment.yaml @@ -73,7 +73,11 @@ spec: - name: flower image: {{ template "flower_image" . }} imagePullPolicy: {{ .Values.images.flower.pullPolicy }} - args: ["bash", "-c", "airflow celery flower || airflow flower"] + {{- if semverCompare ">=2.0.0" .Values.airflowVersion }} + args: ["bash", "-c", "airflow celery flower"] + {{- else }} + args: ["bash", "-c", "airflow flower"] + {{- end }} resources: {{ toYaml .Values.flower.resources | indent 12 }} volumeMounts: diff --git a/chart/templates/jobs/create-user-job.yaml b/chart/templates/jobs/create-user-job.yaml index ea88ef124f22c..a7eada4cf4f06 100644 --- a/chart/templates/jobs/create-user-job.yaml +++ b/chart/templates/jobs/create-user-job.yaml @@ -68,8 +68,11 @@ spec: args: - "bash" - "-c" - # Support running against 1.10.x and 2.x - - 'airflow users create "$@" || airflow create_user "$@"' + {{- if semverCompare ">=2.0.0" .Values.airflowVersion }} + - 'airflow users create "$@"' + {{- else }} + - 'airflow create_user "$@"' + {{- end }} - -- - "-r" - {{ .Values.webserver.defaultUser.role }} diff --git a/chart/templates/jobs/migrate-database-job.yaml b/chart/templates/jobs/migrate-database-job.yaml index 36fbc6db58ae8..e0f0565bf3235 100644 --- a/chart/templates/jobs/migrate-database-job.yaml +++ b/chart/templates/jobs/migrate-database-job.yaml @@ -65,7 +65,11 @@ spec: image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} # Support running against 1.10.x and 2.x - args: ["bash", "-c", "airflow db upgrade || airflow upgradedb"] + {{- if semverCompare ">=2.0.0" .Values.airflowVersion }} + args: ["bash", "-c", "airflow db upgrade"] + {{- else }} + args: ["bash", "-c", "airflow upgradedb"] + {{- end }} envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: diff --git a/chart/templates/pgbouncer/pgbouncer-service.yaml b/chart/templates/pgbouncer/pgbouncer-service.yaml index 6913a4766a97b..fd37b38c4c779 100644 --- a/chart/templates/pgbouncer/pgbouncer-service.yaml +++ b/chart/templates/pgbouncer/pgbouncer-service.yaml @@ -48,9 +48,7 @@ spec: - name: pgbouncer protocol: TCP port: {{ .Values.ports.pgbouncer }} - targetPort: {{ .Values.ports.pgbouncer }} - name: pgbouncer-metrics protocol: TCP port: {{ .Values.ports.pgbouncerScrape }} - targetPort: {{ .Values.ports.pgbouncerScrape }} {{- end }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 54ee0a9d4fbac..fcdc2fbcb4467 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -150,10 +150,12 @@ spec: resources: {{ toYaml .Values.scheduler.resources | indent 12 }} volumeMounts: + {{- if semverCompare ">=1.10.12" .Values.airflowVersion }} - name: config mountPath: {{ include "airflow_pod_template_file" . }}/pod_template_file.yaml subPath: pod_template_file.yaml readOnly: true + {{- end }} - name: logs mountPath: {{ template "airflow_logs" . }} - name: config diff --git a/chart/templates/secrets/metadata-connection-secret.yaml b/chart/templates/secrets/metadata-connection-secret.yaml index c4061feceadb5..cb14a9a42f0b6 100644 --- a/chart/templates/secrets/metadata-connection-secret.yaml +++ b/chart/templates/secrets/metadata-connection-secret.yaml @@ -19,8 +19,8 @@ ## Airflow Metadata Secret ################################# {{- if not .Values.data.metadataSecretName }} -{{- $metadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s.svc.cluster.local" .Release.Name "postgresql" .Release.Namespace) }} -{{- $pgbouncerHost := (printf "%s-%s.%s.svc.cluster.local" .Release.Name "pgbouncer" .Release.Namespace) }} +{{- $metadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} +{{- $pgbouncerHost := (printf "%s-%s.%s" .Release.Name "pgbouncer" .Release.Namespace) }} {{- $host := ternary $pgbouncerHost $metadataHost .Values.pgbouncer.enabled }} {{- $port := ((ternary .Values.ports.pgbouncer .Values.data.metadataConnection.port .Values.pgbouncer.enabled) | toString) }} {{- $database := (ternary (printf "%s-%s" .Release.Name "metadata") .Values.data.metadataConnection.db .Values.pgbouncer.enabled) }} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index ca2369486603a..852e514378a3b 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -37,9 +37,25 @@ metadata: spec: replicas: {{ .Values.webserver.replicas }} strategy: + {{- if .Values.webserver.strategy }} + {{- toYaml .Values.webserver.strategy | nindent 4 }} + {{- else }} + {{- if semverCompare ">=2.0.0" .Values.airflowVersion }} + # Here we define the rolling update strategy + # - maxSurge define how many pod we can add at a time + # - maxUnavailable define how many pod can be unavailable + # during the rolling update + # Setting maxUnavailable to 0 would make sure we have the appropriate + # capacity during the rolling update. + # You can also use percentage based value instead of integer. type: RollingUpdate rollingUpdate: - maxUnavailable: 1 + maxSurge: 1 + maxUnavailable: 0 + {{- else }} + type: Recreate + {{- end}} + {{- end}} selector: matchLabels: tier: airflow diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index b51cedaac6f1e..e48a19c91916a 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -127,7 +127,11 @@ spec: - name: worker image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} - args: ["bash", "-c", "airflow celery worker || airflow worker"] + {{- if semverCompare ">=2.0.0" .Values.airflowVersion }} + args: [ "bash", "-c", "airflow celery worker" ] + {{- else }} + args: [ "bash", "-c", "airflow worker" ] + {{- end }} resources: {{ toYaml .Values.workers.resources | indent 12 }} ports: @@ -158,7 +162,7 @@ spec: subPath: airflow_local_settings.py readOnly: true {{- end }} -{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }} +{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }} - name: dags mountPath: {{ template "airflow_dags_mount_path" . }} {{- end }} diff --git a/chart/tests/conftest.py b/chart/tests/conftest.py index 789f31174135d..7ac27d9be9607 100644 --- a/chart/tests/conftest.py +++ b/chart/tests/conftest.py @@ -19,12 +19,29 @@ import sys import pytest +from filelock import FileLock @pytest.fixture(autouse=True, scope="session") -def upgrade_helm(): +def upgrade_helm(tmp_path_factory, worker_id): """ Upgrade Helm repo """ - subprocess.check_output(["helm", "repo", "add", "stable", "https://charts.helm.sh/stable/"]) - subprocess.check_output(["helm", "dep", "update", sys.path[0]]) + + def _upgrade_helm(): + subprocess.check_output(["helm", "repo", "add", "stable", "https://charts.helm.sh/stable/"]) + subprocess.check_output(["helm", "dep", "update", sys.path[0]]) + + if worker_id == "main": + # not executing in with multiple workers, just update + _upgrade_helm() + return + + root_tmp_dir = tmp_path_factory.getbasetemp().parent + lock_fn = root_tmp_dir / "upgrade_helm.lock" + flag_fn = root_tmp_dir / "upgrade_helm.done" + + with FileLock(str(lock_fn)): + if not flag_fn.is_file(): + _upgrade_helm() + flag_fn.touch() diff --git a/chart/tests/helm_template_generator.py b/chart/tests/helm_template_generator.py index 56a0691f26241..6523d32895d14 100644 --- a/chart/tests/helm_template_generator.py +++ b/chart/tests/helm_template_generator.py @@ -83,16 +83,17 @@ def validate_k8s_object(instance): validate.validate(instance) -def render_chart(name="RELEASE-NAME", values=None, show_only=None): +def render_chart(name="RELEASE-NAME", values=None, show_only=None, chart_dir=None): """ Function that renders a helm chart into dictionaries. For helm chart testing only """ values = values or {} + chart_dir = chart_dir or sys.path[0] with NamedTemporaryFile() as tmp_file: content = yaml.dump(values) tmp_file.write(content.encode()) tmp_file.flush() - command = ["helm", "template", name, sys.path[0], '--values', tmp_file.name] + command = ["helm", "template", name, chart_dir, '--values', tmp_file.name] if show_only: for i in show_only: command.extend(["--show-only", i]) diff --git a/chart/tests/test_flower.py b/chart/tests/test_flower.py index 8c468b47d6e77..18c0db6eeaf91 100644 --- a/chart/tests/test_flower.py +++ b/chart/tests/test_flower.py @@ -44,6 +44,43 @@ def test_create_flower(self, executor, flower_enabled, created): assert "RELEASE-NAME-flower" == jmespath.search("metadata.name", docs[0]) assert "flower" == jmespath.search("spec.template.spec.containers[0].name", docs[0]) + @pytest.mark.parametrize( + "airflow_version, expected_arg", + [ + ( + "2.0.2", + "airflow celery flower", + ), + ( + "1.10.14", + "airflow flower", + ), + ( + "1.9.0", + "airflow flower", + ), + ( + "2.1.0", + "airflow celery flower", + ), + ], + ) + def test_args_with_airflow_version(self, airflow_version, expected_arg): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "flower": {"enabled": True}, + "airflowVersion": airflow_version, + }, + show_only=["templates/flower/flower-deployment.yaml"], + ) + + assert jmespath.search("spec.template.spec.containers[0].args", docs[0]) == [ + "bash", + "-c", + expected_arg, + ] + def test_should_create_flower_deployment_with_authorization(self): docs = render_chart( values={ diff --git a/chart/tests/test_metadata_connection_secret.py b/chart/tests/test_metadata_connection_secret.py index fe90cc0bc259c..0df01901fecc0 100644 --- a/chart/tests/test_metadata_connection_secret.py +++ b/chart/tests/test_metadata_connection_secret.py @@ -53,8 +53,8 @@ def test_default_connection(self): connection = self._get_connection({}) assert ( - "postgresql://postgres:postgres@RELEASE-NAME-postgresql.default.svc.cluster.local:5432" - "/postgres?sslmode=disable" == connection + "postgresql://postgres:postgres@RELEASE-NAME-postgresql.default:5432/postgres?sslmode=disable" + == connection ) def test_should_set_pgbouncer_overrides_when_enabled(self): @@ -63,7 +63,7 @@ def test_should_set_pgbouncer_overrides_when_enabled(self): # host, port, dbname get overridden assert ( - "postgresql://postgres:postgres@RELEASE-NAME-pgbouncer.default.svc.cluster.local:6543" + "postgresql://postgres:postgres@RELEASE-NAME-pgbouncer.default:6543" "/RELEASE-NAME-metadata?sslmode=disable" == connection ) @@ -76,7 +76,7 @@ def test_should_set_pgbouncer_overrides_with_non_chart_database_when_enabled(sel # host, port, dbname still get overridden even with an non-chart db assert ( - "postgresql://someuser:somepass@RELEASE-NAME-pgbouncer.default.svc.cluster.local:6543" + "postgresql://someuser:somepass@RELEASE-NAME-pgbouncer.default:6543" "/RELEASE-NAME-metadata?sslmode=disable" == connection ) diff --git a/chart/tests/test_pgbouncer.py b/chart/tests/test_pgbouncer.py index dbe49c54c8f5c..7677a7b9adb85 100644 --- a/chart/tests/test_pgbouncer.py +++ b/chart/tests/test_pgbouncer.py @@ -15,24 +15,87 @@ # specific language governing permissions and limitations # under the License. +import base64 import unittest import jmespath +from parameterized import parameterized from tests.helm_template_generator import render_chart class PgbouncerTest(unittest.TestCase): + @parameterized.expand(["pgbouncer-deployment", "pgbouncer-service"]) + def test_pgbouncer_resources_not_created_by_default(self, yaml_filename): + docs = render_chart( + show_only=[f"templates/pgbouncer/{yaml_filename}.yaml"], + ) + assert docs == [] + def test_should_create_pgbouncer(self): docs = render_chart( values={"pgbouncer": {"enabled": True}}, show_only=["templates/pgbouncer/pgbouncer-deployment.yaml"], ) + assert "Deployment" == jmespath.search("kind", docs[0]) assert "RELEASE-NAME-pgbouncer" == jmespath.search("metadata.name", docs[0]) - assert "pgbouncer" == jmespath.search("spec.template.spec.containers[0].name", docs[0]) + def test_should_create_pgbouncer_service(self): + docs = render_chart( + values={"pgbouncer": {"enabled": True}}, + show_only=["templates/pgbouncer/pgbouncer-service.yaml"], + ) + + assert "Service" == jmespath.search("kind", docs[0]) + assert "RELEASE-NAME-pgbouncer" == jmespath.search("metadata.name", docs[0]) + assert "true" == jmespath.search('metadata.annotations."prometheus.io/scrape"', docs[0]) + assert "9127" == jmespath.search('metadata.annotations."prometheus.io/port"', docs[0]) + + assert {"prometheus.io/scrape": "true", "prometheus.io/port": "9127"} == jmespath.search( + "metadata.annotations", docs[0] + ) + + assert {"name": "pgbouncer", "protocol": "TCP", "port": 6543} in jmespath.search( + "spec.ports", docs[0] + ) + assert {"name": "pgbouncer-metrics", "protocol": "TCP", "port": 9127} in jmespath.search( + "spec.ports", docs[0] + ) + + def test_pgbouncer_service_with_custom_ports(self): + docs = render_chart( + values={ + "pgbouncer": {"enabled": True}, + "ports": {"pgbouncer": 1111, "pgbouncerScrape": 2222}, + }, + show_only=["templates/pgbouncer/pgbouncer-service.yaml"], + ) + + assert "true" == jmespath.search('metadata.annotations."prometheus.io/scrape"', docs[0]) + assert "2222" == jmespath.search('metadata.annotations."prometheus.io/port"', docs[0]) + assert {"name": "pgbouncer", "protocol": "TCP", "port": 1111} in jmespath.search( + "spec.ports", docs[0] + ) + assert {"name": "pgbouncer-metrics", "protocol": "TCP", "port": 2222} in jmespath.search( + "spec.ports", docs[0] + ) + + def test_pgbouncer_service_extra_annotations(self): + docs = render_chart( + values={ + "pgbouncer": {"enabled": True, "service": {"extraAnnotations": {"foo": "bar"}}}, + }, + show_only=["templates/pgbouncer/pgbouncer-service.yaml"], + ) + + assert { + "prometheus.io/scrape": "true", + "prometheus.io/port": "9127", + "foo": "bar", + } == jmespath.search("metadata.annotations", docs[0]) + def test_should_create_valid_affinity_tolerations_and_node_selector(self): docs = render_chart( values={ @@ -60,7 +123,6 @@ def test_should_create_valid_affinity_tolerations_and_node_selector(self): show_only=["templates/pgbouncer/pgbouncer-deployment.yaml"], ) - assert "Deployment" == jmespath.search("kind", docs[0]) assert "foo" == jmespath.search( "spec.template.spec.affinity.nodeAffinity." "requiredDuringSchedulingIgnoredDuringExecution." @@ -133,3 +195,114 @@ def test_pgbouncer_resources_are_not_added_by_default(self): show_only=["templates/pgbouncer/pgbouncer-deployment.yaml"], ) assert jmespath.search("spec.template.spec.containers[0].resources", docs[0]) == {} + + +class PgbouncerConfigTest(unittest.TestCase): + def test_config_not_created_by_default(self): + docs = render_chart( + show_only=["templates/secrets/pgbouncer-config-secret.yaml"], + ) + + assert docs == [] + + def _get_pgbouncer_ini(self, values: dict) -> str: + docs = render_chart( + values=values, + show_only=["templates/secrets/pgbouncer-config-secret.yaml"], + ) + encoded_ini = jmespath.search('data."pgbouncer.ini"', docs[0]) + return base64.b64decode(encoded_ini).decode() + + def test_databases_default(self): + ini = self._get_pgbouncer_ini({"pgbouncer": {"enabled": True}}) + + assert ( + "RELEASE-NAME-metadata = host=RELEASE-NAME-postgresql.default dbname=postgres port=5432" + " pool_size=10" in ini + ) + assert ( + "RELEASE-NAME-result-backend = host=RELEASE-NAME-postgresql.default dbname=postgres port=5432" + " pool_size=5" in ini + ) + + def test_databases_override(self): + values = { + "pgbouncer": {"enabled": True, "metadataPoolSize": 12, "resultBackendPoolSize": 7}, + "data": { + "metadataConnection": {"host": "meta_host", "db": "meta_db", "port": 1111}, + "resultBackendConnection": {"host": "rb_host", "db": "rb_db", "port": 2222}, + }, + } + ini = self._get_pgbouncer_ini(values) + + assert "RELEASE-NAME-metadata = host=meta_host dbname=meta_db port=1111 pool_size=12" in ini + assert "RELEASE-NAME-result-backend = host=rb_host dbname=rb_db port=2222 pool_size=7" in ini + + def test_config_defaults(self): + ini = self._get_pgbouncer_ini({"pgbouncer": {"enabled": True}}) + + assert "listen_port = 6543" in ini + assert "stats_users = postgres" in ini + assert "max_client_conn = 100" in ini + assert "verbose = 0" in ini + assert "log_disconnections = 0" in ini + assert "log_connections = 0" in ini + assert "server_tls_sslmode = prefer" in ini + assert "server_tls_ciphers = normal" in ini + + assert "server_tls_ca_file = " not in ini + assert "server_tls_cert_file = " not in ini + assert "server_tls_key_file = " not in ini + + def test_config_overrides(self): + values = { + "pgbouncer": { + "enabled": True, + "maxClientConn": 111, + "verbose": 2, + "logDisconnections": 1, + "logConnections": 1, + "sslmode": "verify-full", + "ciphers": "secure", + }, + "ports": {"pgbouncer": 7777}, + "data": {"metadataConnection": {"user": "someuser"}}, + } + ini = self._get_pgbouncer_ini(values) + + assert "listen_port = 7777" in ini + assert "stats_users = someuser" in ini + assert "max_client_conn = 111" in ini + assert "verbose = 2" in ini + assert "log_disconnections = 1" in ini + assert "log_connections = 1" in ini + assert "server_tls_sslmode = verify-full" in ini + assert "server_tls_ciphers = secure" in ini + + def test_ssl_defaults_dont_create_cert_secret(self): + docs = render_chart( + values={"pgbouncer": {"enabled": True}}, + show_only=["templates/secrets/pgbouncer-certificates-secret.yaml"], + ) + + assert docs == [] + + def test_ssl_config(self): + values = { + "pgbouncer": {"enabled": True, "ssl": {"ca": "someca", "cert": "somecert", "key": "somekey"}} + } + ini = self._get_pgbouncer_ini(values) + + assert "server_tls_ca_file = /etc/pgbouncer/root.crt" in ini + assert "server_tls_cert_file = /etc/pgbouncer/server.crt" in ini + assert "server_tls_key_file = /etc/pgbouncer/server.key" in ini + + docs = render_chart( + values=values, + show_only=["templates/secrets/pgbouncer-certificates-secret.yaml"], + ) + + for key, expected in [("root.crt", "someca"), ("server.crt", "somecert"), ("server.key", "somekey")]: + encoded = jmespath.search(f'data."{key}"', docs[0]) + value = base64.b64decode(encoded).decode() + assert expected == value diff --git a/chart/tests/test_pod_template_file.py b/chart/tests/test_pod_template_file.py index 3ec6fb51e59ba..d949ec161cb1f 100644 --- a/chart/tests/test_pod_template_file.py +++ b/chart/tests/test_pod_template_file.py @@ -16,32 +16,36 @@ # under the License. import re +import sys import unittest -from os import remove -from os.path import dirname, realpath -from shutil import copyfile +from shutil import copyfile, copytree +from tempfile import TemporaryDirectory import jmespath +import pytest +from parameterized import parameterized from tests.helm_template_generator import render_chart -ROOT_FOLDER = realpath(dirname(realpath(__file__)) + "/..") - class PodTemplateFileTest(unittest.TestCase): - def setUp(self): - copyfile( - ROOT_FOLDER + "/files/pod-template-file.kubernetes-helm-yaml", - ROOT_FOLDER + "/templates/pod-template-file.yaml", - ) - - def tearDown(self): - remove(ROOT_FOLDER + "/templates/pod-template-file.yaml") + @classmethod + @pytest.fixture(autouse=True, scope="class") + def isolate_chart(cls): + with TemporaryDirectory() as tmp_dir: + cls.temp_chart_dir = tmp_dir + "/chart" + copytree(sys.path[0], cls.temp_chart_dir) + copyfile( + cls.temp_chart_dir + "/files/pod-template-file.kubernetes-helm-yaml", + cls.temp_chart_dir + "/templates/pod-template-file.yaml", + ) + yield def test_should_work(self): docs = render_chart( values={}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -78,6 +82,7 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): }, }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -101,6 +106,59 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "volumeMounts": [{"mountPath": "/git-root", "name": "dags"}], } == jmespath.search("spec.initContainers[0]", docs[0]) + def test_should_not_add_init_container_if_dag_persistence_is_true(self): + docs = render_chart( + values={ + "dags": { + "persistence": {"enabled": True}, + "gitSync": {"enabled": True}, + } + }, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert jmespath.search("spec.initContainers", docs[0]) is None + + @parameterized.expand( + [ + ({"gitSync": {"enabled": True}},), + ({"persistence": {"enabled": True}},), + ( + { + "gitSync": {"enabled": True}, + "persistence": {"enabled": True}, + }, + ), + ] + ) + def test_dags_mount(self, dag_values): + docs = render_chart( + values={"dags": dag_values}, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert {"mountPath": "/opt/airflow/dags", "name": "dags", "readOnly": True} in jmespath.search( + "spec.containers[0].volumeMounts", docs[0] + ) + + def test_dags_mount_with_gitsync_and_persistence(self): + docs = render_chart( + values={ + "dags": { + "gitSync": {"enabled": True}, + "persistence": {"enabled": True}, + } + }, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert {"mountPath": "/opt/airflow/dags", "name": "dags", "readOnly": True} in jmespath.search( + "spec.containers[0].volumeMounts", docs[0] + ) + def test_validate_if_ssh_params_are_added(self): docs = render_chart( values={ @@ -115,6 +173,7 @@ def test_validate_if_ssh_params_are_added(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( @@ -145,6 +204,7 @@ def test_validate_if_ssh_known_hosts_are_added(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"name": "GIT_KNOWN_HOSTS", "value": "true"} in jmespath.search( "spec.initContainers[0].env", docs[0] @@ -171,6 +231,7 @@ def test_should_set_username_and_pass_env_variables(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert { @@ -182,20 +243,50 @@ def test_should_set_username_and_pass_env_variables(self): "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_PASSWORD"}}, } in jmespath.search("spec.initContainers[0].env", docs[0]) - def test_should_set_the_volume_claim_correctly_when_using_an_existing_claim(self): + def test_should_set_the_dags_volume_claim_correctly_when_using_an_existing_claim(self): docs = render_chart( values={"dags": {"persistence": {"enabled": True, "existingClaim": "test-claim"}}}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert {"name": "dags", "persistentVolumeClaim": {"claimName": "test-claim"}} in jmespath.search( "spec.volumes", docs[0] ) + def test_should_use_empty_dir_for_gitsync_without_persistence(self): + docs = render_chart( + values={"dags": {"gitSync": {"enabled": True}}}, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert {"name": "dags", "emptyDir": {}} in jmespath.search("spec.volumes", docs[0]) + + @parameterized.expand( + [ + ({"enabled": False}, {"emptyDir": {}}), + ({"enabled": True}, {"persistentVolumeClaim": {"claimName": "RELEASE-NAME-logs"}}), + ( + {"enabled": True, "existingClaim": "test-claim"}, + {"persistentVolumeClaim": {"claimName": "test-claim"}}, + ), + ] + ) + def test_logs_persistence_changes_volume(self, log_persistence_values, expected): + docs = render_chart( + values={"logs": {"persistence": log_persistence_values}}, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert {"name": "logs", **expected} in jmespath.search("spec.volumes", docs[0]) + def test_should_set_a_custom_image_in_pod_template(self): docs = render_chart( values={"images": {"pod_template": {"repository": "dummy_image", "tag": "latest"}}}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -206,6 +297,7 @@ def test_mount_airflow_cfg(self): docs = render_chart( values={}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -241,6 +333,7 @@ def test_should_create_valid_affinity_and_node_selector(self): "nodeSelector": {"diskType": "ssd"}, }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert re.search("Pod", docs[0]["kind"]) @@ -265,6 +358,7 @@ def test_should_add_fsgroup_to_the_pod_template(self): docs = render_chart( values={"gid": 5000}, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) self.assertEqual(5000, jmespath.search("spec.securityContext.fsGroup", docs[0])) @@ -278,6 +372,7 @@ def test_should_create_valid_volume_mount_and_volume(self): } }, show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, ) assert "test-volume" == jmespath.search( diff --git a/chart/tests/test_webserver_deployment.py b/chart/tests/test_webserver_deployment.py index d89f32d83f9b5..de897f7a0a954 100644 --- a/chart/tests/test_webserver_deployment.py +++ b/chart/tests/test_webserver_deployment.py @@ -240,3 +240,28 @@ def test_webserver_resources_are_not_added_by_default(self): show_only=["templates/webserver/webserver-deployment.yaml"], ) assert jmespath.search("spec.template.spec.containers[0].resources", docs[0]) == {} + + @parameterized.expand( + [ + ("2.0.2", {"type": "RollingUpdate", "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}}), + ("1.10.14", {"type": "Recreate"}), + ("1.9.0", {"type": "Recreate"}), + ("2.1.0", {"type": "RollingUpdate", "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}}), + ], + ) + def test_default_update_strategy(self, airflow_version, expected_strategy): + docs = render_chart( + values={"airflowVersion": airflow_version}, + show_only=["templates/webserver/webserver-deployment.yaml"], + ) + + assert jmespath.search("spec.strategy", docs[0]) == expected_strategy + + def test_update_strategy(self): + expected_strategy = {"type": "RollingUpdate", "rollingUpdate": {"maxUnavailable": 1}} + docs = render_chart( + values={"webserver": {"strategy": expected_strategy}}, + show_only=["templates/webserver/webserver-deployment.yaml"], + ) + + assert jmespath.search("spec.strategy", docs[0]) == expected_strategy diff --git a/chart/values.schema.json b/chart/values.schema.json index 7d6f724c3a687..e6a29ba6412f5 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -31,6 +31,10 @@ "description": "Default airflow tag to deploy.", "type": "string" }, + "airflowVersion": { + "description": "Airflow version (Used to make some decisions based on Airflow Version being deployed).", + "type": "string" + }, "nodeSelector": { "description": "Select certain nodes for airflow pods.", "type": "object", @@ -941,6 +945,10 @@ "description": "How many Airflow webserver replicas should run.", "type": "integer" }, + "strategy": { + "description": "Specifies the strategy used to replace old Pods by new ones.", + "type": ["null", "object"] + }, "serviceAccount": { "description": "Create ServiceAccount.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index bc8cdd311230c..c2b53e78187ca 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -39,6 +39,43 @@ defaultAirflowRepository: apache/airflow # Default airflow tag to deploy defaultAirflowTag: 2.0.2 +# Airflow version (Used to make some decisions based on Airflow Version being deployed) +airflowVersion: 2.0.2 + +# Images +images: + airflow: + repository: ~ + tag: ~ + pullPolicy: IfNotPresent + pod_template: + repository: ~ + tag: ~ + pullPolicy: IfNotPresent + flower: + repository: ~ + tag: ~ + pullPolicy: IfNotPresent + statsd: + repository: apache/airflow + tag: airflow-statsd-exporter-2021.04.28-v0.17.0 + pullPolicy: IfNotPresent + redis: + repository: redis + tag: 6-buster + pullPolicy: IfNotPresent + pgbouncer: + repository: apache/airflow + tag: airflow-pgbouncer-2021.04.28-1.14.0 + pullPolicy: IfNotPresent + pgbouncerExporter: + repository: apache/airflow + tag: airflow-pgbouncer-exporter-2021.04.28-0.5.0 + pullPolicy: IfNotPresent + gitSync: + repository: k8s.gcr.io/git-sync/git-sync + tag: v3.3.0 + pullPolicy: IfNotPresent # Select certain nodes for airflow pods. nodeSelector: {} @@ -126,41 +163,6 @@ executor: "CeleryExecutor" # will be able to launch pods. allowPodLaunching: true -# Images -images: - airflow: - repository: ~ - tag: ~ - pullPolicy: IfNotPresent - pod_template: - repository: ~ - tag: ~ - pullPolicy: IfNotPresent - flower: - repository: ~ - tag: ~ - pullPolicy: IfNotPresent - statsd: - repository: apache/airflow - tag: airflow-statsd-exporter-2021.04.28-v0.17.0 - pullPolicy: IfNotPresent - redis: - repository: redis - tag: 6-buster - pullPolicy: IfNotPresent - pgbouncer: - repository: apache/airflow - tag: airflow-pgbouncer-2021.04.28-1.14.0 - pullPolicy: IfNotPresent - pgbouncerExporter: - repository: apache/airflow - tag: airflow-pgbouncer-exporter-2021.04.28-0.5.0 - pullPolicy: IfNotPresent - gitSync: - repository: k8s.gcr.io/git-sync/git-sync - tag: v3.3.0 - pullPolicy: IfNotPresent - # Environment variables for all airflow containers env: [] # - name: "" @@ -520,6 +522,9 @@ webserver: # Annotations to add to webserver kubernetes service account. annotations: {} + # Allow overriding Update Strategy for Webserver + strategy: ~ + # Additional network policies as needed extraNetworkPolicies: [] @@ -876,7 +881,7 @@ config: # This is ignored when used with the official Docker image load_examples: 'False' executor: '{{ .Values.executor }}' - # For Airflow 1.10, backward compatibility + # For Airflow 1.10, backward compatibility; moved to [logging] in 2.0 colored_console_log: 'False' remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}' # Authentication backend used for the experimental API @@ -890,15 +895,13 @@ config: statsd_port: 9125 statsd_prefix: airflow statsd_host: '{{ printf "%s-statsd" .Release.Name }}' - operators: - default_queue: celery webserver: enable_proxy_fix: 'True' + # For Airflow 1.10 rbac: 'True' celery: worker_concurrency: 16 scheduler: - scheduler_heartbeat_sec: 5 # statsd params included for Airflow 1.10 backward compatibility; moved to [metrics] in 2.0 statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}' statsd_port: 9125 @@ -925,7 +928,6 @@ config: pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml' worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}' worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}' - delete_worker_pods: 'True' multi_namespace_mode: '{{ if .Values.multiNamespaceMode }}True{{ else }}False{{ end }}' # yamllint enable rule:line-length diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 87491fed14615..1b7c41be97b4b 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -55,8 +55,8 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag ```shell script # Set Version - export VERSION=1.10.2rc3 - + export VERSION=2.0.2rc3 + export VERSION_SUFFIX=rc3 # Set AIRFLOW_REPO_ROOT to the path of your git repo export AIRFLOW_REPO_ROOT=$(pwd) @@ -68,7 +68,7 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag export AIRFLOW_REPO_ROOT=$(pwd) ``` -- Set your version to 1.10.2 in `setup.py` (without the RC tag) +- Set your version to 2.0.N in `setup.py` (without the RC tag) - Commit the version change. - Tag your release @@ -80,62 +80,25 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag - Clean the checkout: the sdist step below will ```shell script + rm -rf dist/* git clean -fxd ``` - Tarball the repo ```shell script - git archive --format=tar.gz ${VERSION} --prefix=apache-airflow-${VERSION}/ -o apache-airflow-${VERSION}-source.tar.gz - ``` - - -- Generate sdist - - NOTE: Make sure your checkout is clean at this stage - any untracked or changed files will otherwise be included - in the file produced. - - ```shell script - python setup.py compile_assets sdist bdist_wheel - ``` - -- Rename the sdist - - **Airflow 2+**: - - ```shell script - mv dist/apache-airflow-${VERSION%rc?}.tar.gz apache-airflow-${VERSION}-bin.tar.gz - mv dist/apache_airflow-${VERSION%rc?}-py3-none-any.whl apache_airflow-${VERSION}-py3-none-any.whl + git archive --format=tar.gz ${VERSION} --prefix=apache-airflow-${VERSION}/ -o dist/apache-airflow-${VERSION}-source.tar.gz ``` - **Airflow 1.10.x**: - - ```shell script - mv dist/apache-airflow-${VERSION%rc?}.tar.gz apache-airflow-${VERSION}-bin.tar.gz - mv dist/apache_airflow-${VERSION%rc?}-py2.py3-none-any.whl apache_airflow-${VERSION}-py2.py3-none-any.whl - ``` - Generate SHA512/ASC (If you have not generated a key yet, generate it by following instructions on http://www.apache.org/dev/openpgp.html#key-gen-generate-key) - **Airflow 2+**: - ```shell script - ${AIRFLOW_REPO_ROOT}/dev/sign.sh apache-airflow-${VERSION}-source.tar.gz - ${AIRFLOW_REPO_ROOT}/dev/sign.sh apache-airflow-${VERSION}-bin.tar.gz - ${AIRFLOW_REPO_ROOT}/dev/sign.sh apache_airflow-${VERSION}-py3-none-any.whl + ./breeze prepare-airflow-packages --package-format both --version-suffix-for-svn "${VERSION_SUFFIX}" + ${AIRFLOW_REPO_ROOT}/dev/sign.sh dist/* ``` - **Airflow 1.10.x**: - - ```shell script - ${AIRFLOW_REPO_ROOT}/dev/sign.sh apache-airflow-${VERSION}-source.tar.gz - ${AIRFLOW_REPO_ROOT}/dev/sign.sh apache-airflow-${VERSION}-bin.tar.gz - ${AIRFLOW_REPO_ROOT}/dev/sign.sh apache_airflow-${VERSION}-py2.py3-none-any.whl - ``` - -- Tag & Push latest constraints files. This pushes constraints with rc suffix (this is expected)! - - **Airflow 2+**: +- Tag & Push the latest constraints files. This pushes constraints with rc suffix (this is expected)! ```shell script git checkout constraints-2-0 @@ -143,31 +106,23 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag git push origin "constraints-${VERSION}" ``` - **Airflow 1.10.x**: +- Push the artifacts to ASF dev dist repo ```shell script - git checkout constraints-1-10 - git tag -s "constraints-${VERSION}" - git push origin "constraints-${VERSION}" + # First clone the repo + svn checkout https://dist.apache.org/repos/dist/dev/airflow airflow-dev + + # Create new folder for the release + cd airflow-dev + svn mkdir ${VERSION} + + # Move the artifacts to svn folder & commit + mv ${AIRFLOW_REPO_ROOT}/dist/* ${VERSION}/ + cd ${VERSION} + svn add * + svn commit -m "Add artifacts for Airflow ${VERSION}" ``` -- Push the artifacts to ASF dev dist repo - -``` -# First clone the repo -svn checkout https://dist.apache.org/repos/dist/dev/airflow airflow-dev - -# Create new folder for the release -cd airflow-dev -svn mkdir ${VERSION} - -# Move the artifacts to svn folder & commit -mv ${AIRFLOW_REPO_ROOT}/apache{-,_}airflow-${VERSION}* ${VERSION}/ -cd ${VERSION} -svn add * -svn commit -m "Add artifacts for Airflow ${VERSION}" -``` - ## Prepare PyPI convenience "snapshot" packages At this point we have the artefact that we vote on, but as a convenience to developers we also want to @@ -181,7 +136,7 @@ To do this we need to - Build the package: ```shell script - python setup.py compile_assets egg_info --tag-build "$(sed -e "s/^[0-9.]*//" <<<"$VERSION")" sdist bdist_wheel + ./breeze prepare-airflow-package --version-suffix-for-pypi "${VERSION_SUFFIX}" ``` - Verify the artifacts that would be uploaded: @@ -226,51 +181,26 @@ pushed. If this did not happen - please login to DockerHub and check the status In case you need, you can also build and push the images manually: -### Airflow 2+: - ```shell script -export VERSION_RC= +export VERSION= export DOCKER_REPO=docker.io/apache/airflow for python_version in "3.6" "3.7" "3.8" ( - export DOCKER_TAG=${VERSION_RC}-python${python_version} + export DOCKER_TAG=${VERSION}-python${python_version} ./scripts/ci/images/ci_build_dockerhub.sh ) ``` -Once this succeeds you should push the "${VERSION_RC}" image: +Once this succeeds you should push the "${VERSION}" image: ```shell script -docker tag apache/airflow:${VERSION_RC}-python3.6 apache/airflow:${VERSION_RC} -docker push apache/airflow:${VERSION_RC} +docker tag apache/airflow:${VERSION}-python3.6 apache/airflow:${VERSION} +docker push apache/airflow:${VERSION} ``` This will wipe Breeze cache and docker-context-files in order to make sure the build is "clean". It also performs image verification before the images are pushed. - -### Airflow 1.10: - -```shell script -for python_version in "2.7" "3.5" "3.6" "3.7" "3.8" -do - ./breeze build-image --production-image --python ${python_version} \ - --image-tag apache/airflow:${VERSION_RC}-python${python_version} --build-cache-local - docker push apache/airflow:${VERSION_RC}-python${python_version} -done -``` - -Once this succeeds you should push the "${VERSION_RC}" image: - -```shell script -docker tag apache/airflow:${VERSION_RC}-python3.6 apache/airflow:${VERSION_RC} -docker push apache/airflow:${VERSION_RC} -``` - - -### Airflow 1.10: - - ## Prepare Vote email on the Apache Airflow release candidate - Use the dev/airflow-jira script to generate a list of Airflow JIRAs that were closed in the release. @@ -280,7 +210,7 @@ docker push apache/airflow:${VERSION_RC} Subject: ``` -[VOTE] Airflow 1.10.2rc3 +[VOTE] Airflow 2.0.2rc3 ``` Body: @@ -288,15 +218,15 @@ Body: ``` Hey all, -I have cut Airflow 1.10.2 RC3. This email is calling a vote on the release, +I have cut Airflow 2.0.2 RC3. This email is calling a vote on the release, which will last for 72 hours. Consider this my (binding) +1. -Airflow 1.10.2 RC3 is available at: -https://dist.apache.org/repos/dist/dev/airflow/1.10.2rc3/ +Airflow 2.0.2 RC3 is available at: +https://dist.apache.org/repos/dist/dev/airflow/2.0.2rc3/ -*apache-airflow-1.10.2rc3-source.tar.gz* is a source release that comes +*apache-airflow-2.0.2rc3-source.tar.gz* is a source release that comes with INSTALL instructions. -*apache-airflow-1.10.2rc3-bin.tar.gz* is the binary Python "sdist" release. +*apache-airflow-2.0.2rc3-bin.tar.gz* is the binary Python "sdist" release. Public keys are available at: https://dist.apache.org/repos/dist/release/airflow/KEYS @@ -308,11 +238,11 @@ The test procedure for PMCs and Contributors who would like to test this RC are https://github.com/apache/airflow/blob/master/dev/README.md#vote-and-verify-the-apache-airflow-release-candidate Please note that the version number excludes the `rcX` string, so it's now -simply 1.10.2. This will allow us to rename the artifact without modifying +simply 2.0.2. This will allow us to rename the artifact without modifying the artifact checksums when we actually release. -Changes since 1.10.2rc2: +Changes since 2.0.2rc2: *Bugs*: [AIRFLOW-3732] Fix issue when trying to edit connection in RBAC UI [AIRFLOW-2866] Fix missing CSRF token head when using RBAC UI (#3804) @@ -447,24 +377,24 @@ warning. By importing the server in the previous step and importing it via ID fr this is a valid Key already. ``` -Checking apache-airflow-1.10.12rc4-bin.tar.gz.asc -gpg: assuming signed data in 'apache-airflow-1.10.12rc4-bin.tar.gz' +Checking apache-airflow-2.0.2rc4-bin.tar.gz.asc +gpg: assuming signed data in 'apache-airflow-2.0.2rc4-bin.tar.gz' gpg: Signature made sob, 22 sie 2020, 20:28:28 CEST gpg: using RSA key 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B gpg: Good signature from "Kaxil Naik " [unknown] gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. Primary key fingerprint: 1271 7556 040E EF2E EAF1 B9C2 75FC CD0A 25FA 0E4B -Checking apache_airflow-1.10.12rc4-py2.py3-none-any.whl.asc -gpg: assuming signed data in 'apache_airflow-1.10.12rc4-py2.py3-none-any.whl' +Checking apache_airflow-2.0.2rc4-py2.py3-none-any.whl.asc +gpg: assuming signed data in 'apache_airflow-2.0.2rc4-py2.py3-none-any.whl' gpg: Signature made sob, 22 sie 2020, 20:28:31 CEST gpg: using RSA key 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B gpg: Good signature from "Kaxil Naik " [unknown] gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. Primary key fingerprint: 1271 7556 040E EF2E EAF1 B9C2 75FC CD0A 25FA 0E4B -Checking apache-airflow-1.10.12rc4-source.tar.gz.asc -gpg: assuming signed data in 'apache-airflow-1.10.12rc4-source.tar.gz' +Checking apache-airflow-2.0.2rc4-source.tar.gz.asc +gpg: assuming signed data in 'apache-airflow-2.0.2rc4-source.tar.gz' gpg: Signature made sob, 22 sie 2020, 20:28:25 CEST gpg: using RSA key 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B gpg: Good signature from "Kaxil Naik " [unknown] @@ -487,9 +417,9 @@ done You should get output similar to: ``` -Checking apache-airflow-1.10.12rc4-bin.tar.gz.sha512 -Checking apache_airflow-1.10.12rc4-py2.py3-none-any.whl.sha512 -Checking apache-airflow-1.10.12rc4-source.tar.gz.sha512 +Checking apache-airflow-2.0.2rc4-bin.tar.gz.sha512 +Checking apache_airflow-2.0.2rc4-py2.py3-none-any.whl.sha512 +Checking apache-airflow-2.0.2rc4-source.tar.gz.sha512 ``` # Verify release candidates by Contributors @@ -497,7 +427,7 @@ Checking apache-airflow-1.10.12rc4-source.tar.gz.sha512 This can be done (and we encourage to) by any of the Contributors. In fact, it's best if the actual users of Apache Airflow test it in their own staging/test installations. Each release candidate is available on PyPI apart from SVN packages, so everyone should be able to install -the release candidate version of Airflow via simply ( is 1.10.12 for example, and is +the release candidate version of Airflow via simply ( is 2.0.2 for example, and is release candidate number 1,2,3,....). ```shell script @@ -523,12 +453,6 @@ Running the following command will use tmux inside breeze, create `admin` user a ./breeze start-airflow --use-airflow-version rc --python 3.7 --backend postgres ``` -For 1.10 releases you can also use `--no-rbac-ui` flag disable RBAC UI of Airflow: - -```shell script -./breeze start-airflow --use-airflow-version rc --python 3.7 --backend postgres --no-rbac-ui -``` - Once you install and run Airflow, you should perform any verification you see as necessary to check that the Airflow works as you expected. @@ -541,7 +465,7 @@ Once the vote has been passed, you will need to send a result vote to dev@airflo Subject: ``` -[RESULT][VOTE] Airflow 1.10.2rc3 +[RESULT][VOTE] Airflow 2.0.2rc3 ``` Message: @@ -549,7 +473,7 @@ Message: ``` Hello, -Apache Airflow 1.10.2 (based on RC3) has been accepted. +Apache Airflow 2.0.2 (based on RC3) has been accepted. 4 “+1” binding votes received: - Kaxil Naik (binding) @@ -584,25 +508,42 @@ https://dist.apache.org/repos/dist/release/airflow/ The best way of doing this is to svn cp between the two repos (this avoids having to upload the binaries again, and gives a clearer history in the svn commit logs): ```shell script -# First clone the repo -export RC=1.10.4rc5 +# GO to Airflow Sources first +cd +export AIRFLOW_SOURCES=$(pwd) + +# GO to Checked out DEV repo. Should be checked out before via: +# svn checkout https://dist.apache.org/repos/dist/dev/airflow airflow-release +cd +svn update +export AIRFLOW_DEV_SVN=$(pwd) + +# GO to Checked out RELEASE repo. Should be checked out before via: +# svn checkout https://dist.apache.org/repos/dist/dev/airflow airflow-release +cd +svn update + +export RC=2.0.2rc5 export VERSION=${RC/rc?/} -svn checkout https://dist.apache.org/repos/dist/release/airflow airflow-release # Create new folder for the release cd airflow-release -svn mkdir ${VERSION} -cd ${VERSION} +svn mkdir "${VERSION}" +cd "${VERSION}" # Move the artifacts to svn folder & commit -for f in ../../airflow-dev/$RC/*; do svn cp $f ${$(basename $f)/rc?/}; done +for f in ${AIRFLOW_DEV_SVN}/$RC/*; do + svn cp "$f" "${$(basename $f)/rc?/}" + # Those will be used to upload to PyPI + cp "$f" "${AIRFLOW_SOURCES}/dist/${$(basename $f)/rc?/}" +done svn commit -m "Release Airflow ${VERSION} from ${RC}" # Remove old release -# http://www.apache.org/legal/release-policy.html#when-to-archive +# See http://www.apache.org/legal/release-policy.html#when-to-archive cd .. -export PREVIOUS_VERSION=1.10.1 -svn rm ${PREVIOUS_VERSION} +export PREVIOUS_VERSION=2.0.2 +svn rm "${PREVIOUS_VERSION}" svn commit -m "Remove old release: ${PREVIOUS_VERSION}" ``` @@ -610,17 +551,13 @@ Verify that the packages appear in [airflow](https://dist.apache.org/repos/dist/ ## Prepare PyPI "release" packages -At this point we release an official package: - -- Build the package: - - ```shell script - python setup.py compile_assets sdist bdist_wheel - ``` +At this point we release an official package (they should be copied and renamed from the +previously released RC candidates in "${AIRFLOW_SOURCES}/dist": - Verify the artifacts that would be uploaded: ```shell script + cd "${AIRFLOW_SOURCES}" twine check dist/* ``` @@ -692,25 +629,6 @@ for python_version in "3.6" "3.7" "3.8" This will wipe Breeze cache and docker-context-files in order to make sure the build is "clean". It also performs image verification before the images are pushed. - -### Airflow 1.10: - -```shell script -for python_version in "2.7" "3.5" "3.6" "3.7" "3.8" -do - ./breeze build-image --production-image --python ${python_version} \ - --image-tag apache/airflow:${VERSION}-python${python_version} --build-cache-local - docker push apache/airflow:${VERSION}-python${python_version} -done -``` - -Once this succeeds you should push the "${VERSION}" image: - -```shell script -docker tag apache/airflow:${VERSION}-python3.6 apache/airflow:${VERSION} -docker push apache/airflow:${VERSION} -``` - ## Publish documentation Documentation is an essential part of the product and should be made available to users. diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index ba912ee3c7a8d..1b20b0841ac61 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -301,12 +301,13 @@ apache-airflow with doc extra: * `pip install apache-airflow[doc]` -All providers: +All providers (including overriding documentation for doc-only changes): ```shell script ./docs/publish_docs.py \ --package-filter apache-airflow-providers \ - --package-filter 'apache-airflow-providers-*' + --package-filter 'apache-airflow-providers-*' \ + --override-versioned cd "${AIRFLOW_SITE_DIRECTORY}" ``` @@ -508,24 +509,24 @@ warning. By importing the server in the previous step and importing it via ID fr this is a valid Key already. ``` -Checking apache-airflow-1.10.12rc4-bin.tar.gz.asc -gpg: assuming signed data in 'apache-airflow-1.10.12rc4-bin.tar.gz' +Checking apache-airflow-2.0.2rc4-bin.tar.gz.asc +gpg: assuming signed data in 'apache-airflow-2.0.2rc4-bin.tar.gz' gpg: Signature made sob, 22 sie 2020, 20:28:28 CEST gpg: using RSA key 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B gpg: Good signature from "Kaxil Naik " [unknown] gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. Primary key fingerprint: 1271 7556 040E EF2E EAF1 B9C2 75FC CD0A 25FA 0E4B -Checking apache_airflow-1.10.12rc4-py2.py3-none-any.whl.asc -gpg: assuming signed data in 'apache_airflow-1.10.12rc4-py2.py3-none-any.whl' +Checking apache_airflow-2.0.2rc4-py2.py3-none-any.whl.asc +gpg: assuming signed data in 'apache_airflow-2.0.2rc4-py2.py3-none-any.whl' gpg: Signature made sob, 22 sie 2020, 20:28:31 CEST gpg: using RSA key 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B gpg: Good signature from "Kaxil Naik " [unknown] gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. Primary key fingerprint: 1271 7556 040E EF2E EAF1 B9C2 75FC CD0A 25FA 0E4B -Checking apache-airflow-1.10.12rc4-source.tar.gz.asc -gpg: assuming signed data in 'apache-airflow-1.10.12rc4-source.tar.gz' +Checking apache-airflow-2.0.2rc4-source.tar.gz.asc +gpg: assuming signed data in 'apache-airflow-2.0.2rc4-source.tar.gz' gpg: Signature made sob, 22 sie 2020, 20:28:25 CEST gpg: using RSA key 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B gpg: Good signature from "Kaxil Naik " [unknown] diff --git a/dev/provider_packages/README.md b/dev/provider_packages/README.md index a8da8edd623e4..61b9b4ec12dca 100644 --- a/dev/provider_packages/README.md +++ b/dev/provider_packages/README.md @@ -209,7 +209,7 @@ of those steps automatically, but you can manually run the scripts as follows t The commands are best to execute in the Breeze environment as it has all the dependencies installed, Examples below describe that. However, for development you might run them in your local development environment as it makes it easier to debug. Just make sure you install your development environment -with 'devel_all' extra (make sure to ue the right python version). +with 'devel_all' extra (make sure to use the right python version). Note that it is best to use `INSTALL_PROVIDERS_FROM_SOURCES` set to`true`, to make sure that any new added providers are not added as packages (in case they are not yet available in PyPI. diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index cd0d41bb99ce0..cd18b2c8462ec 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -348,7 +348,7 @@ def get_install_requirements(provider_package_id: str) -> List[str]: :return: install requirements of the package """ dependencies = PROVIDERS_REQUIREMENTS[provider_package_id] - airflow_dependency = 'apache-airflow>=2.0.0' + airflow_dependency = 'apache-airflow>=2.1.0.dev0' # Avoid circular dependency for the preinstalled packages install_requires = [airflow_dependency] if provider_package_id not in PREINSTALLED_PROVIDERS else [] install_requires.extend(dependencies) @@ -1639,7 +1639,7 @@ def update_setup_files( :param provider_package_id: id of the package :param version_suffix: version suffix corresponding to the version in the code - :returns False if the package should be skipped, Tre if everything generated properly + :returns False if the package should be skipped, True if everything generated properly """ verify_provider_package(provider_package_id) provider_details = get_provider_details(provider_package_id) diff --git a/docs/apache-airflow-providers-apache-hive/connections/hive_cli.rst b/docs/apache-airflow-providers-apache-hive/connections/hive_cli.rst new file mode 100644 index 0000000000000..da984ee3a539b --- /dev/null +++ b/docs/apache-airflow-providers-apache-hive/connections/hive_cli.rst @@ -0,0 +1,93 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/connection:hive_cli: + +Hive CLI Connection +=================== + +The Hive CLI connection type enables the Hive CLI Integrations. + +Authenticating to Hive CLI +-------------------------- + +There are two ways to connect to Hive using Airflow. + +1. Use the `Hive Beeline + `_. + i.e. make a jdbc connection string with host, port, and schema. Optionally you can connect with a proxy user, and specify a login and password. + +2. Use the `Hive CLI + `_. + i.e. specify hive cli params in the extras field. + +Only one authorization method can be used at a time. If you need to manage multiple credentials or keys then you should +configure multiple connections. + +Default Connection IDs +---------------------- + +All hooks and operators related to Hive_CLI use ``hive_cli_default`` by default. + +Configuring the Connection +-------------------------- + +Login (optional) + Specify your username for a proxy user or for the Beeline CLI. + +Password (optional) + Specify your Beeline CLI password. + +Host (optional) + Specify your jdbc hive host that is used for Hive Beeline. + +Port (optional) + Specify your jdbc hive port that is used for Hive Beeline. + +Schema (optional) + Specify your jdbc hive database that you want to connect to with Beeline + or specify a schema for an HQL statement to run with the Hive CLI. + +Extra (optional) + Specify the extra parameters (as json dictionary) that can be used in Hive CLI connection. + The following parameters are all optional: + + * ``hive_cli_params`` + Specify an object CLI params for use with Beeline CLI and Hive CLI. + * ``use_beeline`` + Specify as ``True`` if using the Beeline CLI. Default is ``False``. + * ``auth`` + Specify the auth type for use with Hive Beeline CLI. + * ``proxy_user`` + Specify a proxy user as an ``owner`` or ``login`` or keep blank if using a + custom proxy user. + * ``principal`` + Specify the jdbc hive principal to be used with Hive Beeline. + + +When specifying the connection in environment variable you should specify +it using URI syntax. + +Note that all components of the URI should be URL-encoded. + +For example: + +.. code-block:: bash + + export AIRFLOW_CONN_HIVE_CLI_DEFAULT='hive-cli://beeline-username:beeline-password@jdbc-hive-host:80/hive-database?hive_cli_params=params&use_beeline=True&auth=noSasl&principal=hive%2F_HOST%40EXAMPLE.COM' diff --git a/docs/apache-airflow-providers-apache-hive/connections/hive_metastore.rst b/docs/apache-airflow-providers-apache-hive/connections/hive_metastore.rst new file mode 100644 index 0000000000000..b0445521acb51 --- /dev/null +++ b/docs/apache-airflow-providers-apache-hive/connections/hive_metastore.rst @@ -0,0 +1,69 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/connection:hive_metastore: + +Hive Metastore Connection +========================= + +The Hive Metastore connection type enables the Hive Metastore Integrations. + +Authenticating to Hive Metastore +-------------------------------- + +Authentication with the Hive Metastore through `Apache Thrift Hive Server +`_ +and the `hmsclient +`_. + + +Default Connection IDs +---------------------- + +All hooks and operators related to the Hive Metastore use ``metastore_default`` by default. + +Configuring the Connection +-------------------------- + +Host (optional) + The host of your Hive Metastore node. + +Port (optional) + Your Hive Metastore port number. + +Extra (optional) + Specify the extra parameters (as json dictionary) that can be used in Hive Metastore connection. + The following parameters are all optional: + + * ``auth_mechanism`` + Specify the mechanism for authentication the default is ``NOSASL``. + * ``kerberos_service_name`` + Specify The kerberos service name the default is ``hive``. + + +When specifying the connection in environment variable you should specify +it using URI syntax. + +Note that all components of the URI should be URL-encoded. + +For example: + +.. code-block:: bash + + export AIRFLOW_CONN_METASTORE_DEFAULT='hive-metastore://hive-metastore-node:80?auth_mechanism=NOSASL' diff --git a/docs/apache-airflow-providers-apache-hive/connections/hiveserver2.rst b/docs/apache-airflow-providers-apache-hive/connections/hiveserver2.rst new file mode 100644 index 0000000000000..c5ae2bfa63703 --- /dev/null +++ b/docs/apache-airflow-providers-apache-hive/connections/hiveserver2.rst @@ -0,0 +1,78 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/connection:hiveserver2: + +Hive Server2 Connection +========================= + +The Hive Server2 connection type enables the Hive Server2 Integrations. + +Authenticating to Hive Server2 +------------------------------ + +Connect to Hive Server2 using `PyHive +`_. +Choose between authenticating via LDAP, kerberos, or custom. + +Default Connection IDs +---------------------- + +All hooks and operators related to Hive Server2 use ``hiveserver2_default`` by default. + +Configuring the Connection +-------------------------- + +Login (optional) + Specify your Hive Server2 username. + +Password (optional) + Specify your Hive password for use with LDAP and custom authentication. + +Host (optional) + Specify the host node for Hive Server2. + +Port (optional) + Specify your Hive Server2 port number. + +Schema (optional) + Specify the name for the database you would like to connect to with Hive Server2. + +Extra (optional) + Specify the extra parameters (as json dictionary) that can be used in Hive Server2 connection. + The following parameters are all optional: + + * ``auth_mechanism`` + Specify the authentication method for PyHive choose between ``PLAIN``, ``LDAP``, ``KERBEROS`` or, ``Custom`` the default is ``PLAIN``. + * ``kerberos_service_name`` + If authenticating with kerberos specify the kerberos service name the default is ``hive``. + * ``run_set_variable_statements`` + Specify the if you want to run set variable statements the default is ```True``. + + +When specifying the connection in environment variable you should specify +it using URI syntax. + +Note that all components of the URI should be URL-encoded. + +For example: + +.. code-block:: bash + + export AIRFLOW_CONN_HIVESERVER2_DEFAULT='hiveserver2://username:password@hiveserver2-node:80/database?auth_mechanism=LDAP' diff --git a/docs/apache-airflow-providers-apache-hive/connections/index.rst b/docs/apache-airflow-providers-apache-hive/connections/index.rst new file mode 100644 index 0000000000000..2b8afb41df936 --- /dev/null +++ b/docs/apache-airflow-providers-apache-hive/connections/index.rst @@ -0,0 +1,25 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Connection Types +---------------- + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/apache-airflow-providers-apache-hive/index.rst b/docs/apache-airflow-providers-apache-hive/index.rst index f25293a17e832..ec706330a03e4 100644 --- a/docs/apache-airflow-providers-apache-hive/index.rst +++ b/docs/apache-airflow-providers-apache-hive/index.rst @@ -22,6 +22,12 @@ Content ------- +.. toctree:: + :maxdepth: 1 + :caption: Guides + + Connection types + .. toctree:: :maxdepth: 1 :caption: References diff --git a/docs/apache-airflow-providers-google/api-auth-backend/google-openid.rst b/docs/apache-airflow-providers-google/api-auth-backend/google-openid.rst index f1046cc99b455..95f3a4cbf54c0 100644 --- a/docs/apache-airflow-providers-google/api-auth-backend/google-openid.rst +++ b/docs/apache-airflow-providers-google/api-auth-backend/google-openid.rst @@ -57,7 +57,7 @@ look like the following. .. code-block:: bash - ENDPOINT_URL="http://locahost:8080/" + ENDPOINT_URL="http://localhost:8080/" AUDIENCE="project-id-random-value.apps.googleusercontent.com" ID_TOKEN="$(gcloud auth print-identity-token "--audience=${AUDIENCE}")" diff --git a/docs/apache-airflow-providers-google/connections/gcp.rst b/docs/apache-airflow-providers-google/connections/gcp.rst index b37e251e69c3a..7cdc8d105667d 100644 --- a/docs/apache-airflow-providers-google/connections/gcp.rst +++ b/docs/apache-airflow-providers-google/connections/gcp.rst @@ -180,7 +180,8 @@ For example, with the following ``terraform`` setup... terraform { required_version = "> 0.11.14" } - provider "google" {} + provider "google" { + } variable "project_id" { type = "string" } diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst index 7fc27804db38d..b99971a193065 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst @@ -245,6 +245,23 @@ in the given dataset. :start-after: [START howto_operator_bigquery_upsert_table] :end-before: [END howto_operator_bigquery_upsert_table] +.. _howto/operator:BigQueryUpdateTableSchemaOperator: + +Update table schema +""""""""""""""""""" + +To update the schema of a table you can use +:class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryUpdateTableSchemaOperator`. + +This operator updates the schema field values supplied, while leaving the rest unchanged. This is useful +for instance to set new field descriptions on an existing table schema. + +.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_bigquery_update_table_schema] + :end-before: [END howto_operator_bigquery_update_table_schema] + .. _howto/operator:BigQueryDeleteTableOperator: Delete table diff --git a/docs/apache-airflow-providers-imap/connections/imap.rst b/docs/apache-airflow-providers-imap/connections/imap.rst index ad53ee0a3aca0..bf702a3bd5289 100644 --- a/docs/apache-airflow-providers-imap/connections/imap.rst +++ b/docs/apache-airflow-providers-imap/connections/imap.rst @@ -43,7 +43,7 @@ Login Specify the username used for the IMAP client. Password - Specify the password used fot the IMAP client. + Specify the password used for the IMAP client. Host Specify the the IMAP host url. diff --git a/docs/apache-airflow-providers-oracle/connections/oracle.rst b/docs/apache-airflow-providers-oracle/connections/oracle.rst index 4057d22a289fa..4a26c36b3142a 100644 --- a/docs/apache-airflow-providers-oracle/connections/oracle.rst +++ b/docs/apache-airflow-providers-oracle/connections/oracle.rst @@ -17,31 +17,25 @@ +.. _howto/connection:oracle: + Oracle Connection ================= The Oracle connection type provides connection to a Oracle database. Configuring the Connection -------------------------- -Dsn (required) - The Data Source Name. The host address for the Oracle server. - -Host(optional) - Connect descriptor string for the data source name. - -Sid (optional) - The Oracle System ID. The uniquely identify a particular database on a system. -Service_name (optional) - The db_unique_name of the database. +Host (optional) + The host to connect to. -Port (optional) - The port for the Oracle server, Default ``1521``. +Schema (optional) + Specify the schema name to be used in the database. -Login (required) +Login (optional) Specify the user name to connect. -Password (required) +Password (optional) Specify the password to connect. Extra (optional) @@ -62,8 +56,10 @@ Extra (optional) which are defined at the module level, Default mode is connecting. * ``purity`` - one of ``new``, ``self``, ``default``. Specify the session acquired from the pool. configuration parameter. + * ``dsn``. Specify a Data Source Name (and ignore Host). + * ``sid`` or ``service_name``. Use to form DSN instead of Schema. - Connect using Dsn and Sid, Dsn and Service_name, or only Host `(OracleHook.getconn Documentation) `_. + Connect using `dsn`, Host and `sid`, Host and `service_name`, or only Host `(OracleHook.getconn Documentation) `_. For example: @@ -75,15 +71,15 @@ Extra (optional) .. code-block:: python - Dsn = "dbhost.example.com" - Service_name = "orclpdb1" + Host = "dbhost.example.com" + Schema = "orclpdb1" or .. code-block:: python - Dsn = "dbhost.example.com" - Sid = "orcl" + Host = "dbhost.example.com" + Schema = "orcl" More details on all Oracle connect parameters supported can be found in `cx_Oracle documentation diff --git a/docs/apache-airflow-providers/howto/create-update-providers.rst b/docs/apache-airflow-providers/howto/create-update-providers.rst index 47ebb77c1475b..91fb74e438d6a 100644 --- a/docs/apache-airflow-providers/howto/create-update-providers.rst +++ b/docs/apache-airflow-providers/howto/create-update-providers.rst @@ -34,7 +34,7 @@ help you to set up tests and other dependencies. First, you need to set up your local development environment. See `Contribution Quick Start `_ if you did not set up your local environment yet. We recommend using ``breeze`` to develop locally. This way you -easily be able to have an environment more similar to the one executed by Github CI workflow. +easily be able to have an environment more similar to the one executed by GitHub CI workflow. .. code-block:: bash @@ -55,7 +55,7 @@ Most likely you have developed a version of the provider using some local custom transfer this code to the Airflow project. Below is described all the initial code structure that the provider may need. Understand that not all providers will need all the components described in this structure. If you still have doubts about building your provider, we recommend that you read the initial provider guide and -open a issue on Github so the community can help you. +open a issue on GitHub so the community can help you. .. code-block:: bash diff --git a/docs/apache-airflow/concepts/dags.rst b/docs/apache-airflow/concepts/dags.rst index b82b55f5c1fc1..46849ab9de426 100644 --- a/docs/apache-airflow/concepts/dags.rst +++ b/docs/apache-airflow/concepts/dags.rst @@ -585,3 +585,21 @@ Note that packaged DAGs come with some caveats: * They will be inserted into Python's ``sys.path`` and importable by any other code in the Airflow process, so ensure the package names don't clash with other packages already installed on your system. In general, if you have a complex set of compiled dependencies and modules, you are likely better off using the Python ``virtualenv`` system and installing the necessary packages on your target systems with ``pip``. + +DAG Dependencies +================ + +*Added in Airflow 2.1* + +While dependencies between tasks in a DAG are explicitly defined through upstream and downstream +relationships, dependencies between DAGs are a bit more complex. In general, there are two ways +in which one DAG can depend on another: + +- triggering - :class:`~airflow.operators.trigger_dagrun.TriggerDagRunOperator` +- waiting - :class:`~airflow.sensors.external_task_sensor.ExternalTaskSensor` + +Additional difficulty is that one DAG could wait for or trigger several runs of the other DAG +with different execution dates. The **Dag Dependencies** view +``Menu -> Browse -> DAG Dependencies`` helps visualize dependencies between DAGs. The dependencies +are calculated by the scheduler during DAG serialization and the webserver uses them to build +the dependency graph. diff --git a/docs/apache-airflow/concepts/operators.rst b/docs/apache-airflow/concepts/operators.rst index f7eb6cf293deb..251cedab0ded0 100644 --- a/docs/apache-airflow/concepts/operators.rst +++ b/docs/apache-airflow/concepts/operators.rst @@ -74,7 +74,7 @@ For example, say you want to pass the execution date as an environment variable Here, ``{{ ds }}`` is a macro, and because the ``env`` parameter of the ``BashOperator`` is templated with Jinja, the execution date will be available as an environment variable named ``EXECUTION_DATE`` in your Bash script. -You can use Jinja templating with every parameter that is marked as "templated" in the documentation. Template substitution occurs just before the pre_execute function of your operator is called. +You can use Jinja templating with every parameter that is marked as "templated" in the documentation. Template substitution occurs just before the ``pre_execute`` function of your operator is called. You can also use Jinja templating with nested fields, as long as these nested fields are marked as templated in the structure they belong to: fields registered in ``template_fields`` property will be submitted to template substitution, like the ``path`` field in the example below: @@ -141,3 +141,61 @@ You can pass custom options to the Jinja ``Environment`` when creating your DAG. ) See the `Jinja documentation `_ to find all available options. + +Rendering Fields as Native Python Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, all the ``template_fields`` are rendered as strings. + +Example, let's say ``extract`` task pushes a dictionary +(Example: ``{"1001": 301.27, "1002": 433.21, "1003": 502.22}``) to :ref:`XCom ` table. +Now, when the following task is run, ``order_data`` argument is passed a string, example: +``'{"1001": 301.27, "1002": 433.21, "1003": 502.22}'``. + +.. code-block:: python + + transform = PythonOperator( + task_id="transform", op_kwargs={"order_data": "{{ti.xcom_pull('extract')}}"}, + python_callable=transform + ) + + +If you instead want the rendered template field to return a Native Python object (``dict`` in our example), +you can pass ``render_template_as_native_obj=True`` to the DAG as follows: + +.. code-block:: python + + dag = DAG( + dag_id="example_template_as_python_object", + schedule_interval=None, + start_date=days_ago(2), + render_template_as_native_obj=True, + ) + + def extract(): + data_string = '{"1001": 301.27, "1002": 433.21, "1003": 502.22}' + return json.loads(data_string) + + def transform(order_data): + print(type(order_data)) + for value in order_data.values(): + total_order_value += value + return {"total_order_value": total_order_value} + + extract_task = PythonOperator( + task_id="extract", + python_callable=extract + ) + + transform_task = PythonOperator( + task_id="transform", op_kwargs={"order_data": "{{ti.xcom_pull('extract')}}"}, + python_callable=transform + ) + + extract_task >> transform_task + +In this case, ``order_data`` argument is passed: ``{"1001": 301.27, "1002": 433.21, "1003": 502.22}``. + +Airflow uses Jinja's `NativeEnvironment `_ +when ``render_template_as_native_obj`` is set to ``True``. +With ``NativeEnvironment``, rendering a template produces a native Python type. diff --git a/docs/apache-airflow/rest-api-ref.rst b/docs/apache-airflow/deprecated-rest-api-ref.rst similarity index 90% rename from docs/apache-airflow/rest-api-ref.rst rename to docs/apache-airflow/deprecated-rest-api-ref.rst index 07f85f892cb85..11ec8a7dd0b2b 100644 --- a/docs/apache-airflow/rest-api-ref.rst +++ b/docs/apache-airflow/deprecated-rest-api-ref.rst @@ -15,20 +15,21 @@ specific language governing permissions and limitations under the License. -Experimental REST API Reference -=============================== - -Airflow exposes an REST API. It is available through the webserver. Endpoints are -available at ``/api/experimental/``. +Deprecated REST API +=================== .. warning:: - This REST API is deprecated since version 2.0. Please consider using :doc:`the stable REST API `. + This REST API is deprecated since version 2.0. Please consider using the :doc:`stable REST API `. For more information on migration, see `UPDATING.md `_ +Before Airflow 2.0 this REST API was known as the "experimental" API, but now that the :doc:`stable REST API ` is available, it has been renamed. + +The endpoints for this API are available at ``/api/experimental/``. + .. versionchanged:: 2.0 - The experimental REST API is disabled by default. To restore these APIs while migrating to + This REST API is disabled by default. To restore these APIs while migrating to the stable REST API, set ``enable_experimental_api`` option in ``[api]`` section to ``True``. Endpoints diff --git a/docs/apache-airflow/howto/custom-operator.rst b/docs/apache-airflow/howto/custom-operator.rst index bca1fc76dbd78..e0ad193a33998 100644 --- a/docs/apache-airflow/howto/custom-operator.rst +++ b/docs/apache-airflow/howto/custom-operator.rst @@ -28,8 +28,7 @@ You can create any operator you want by extending the :class:`airflow.models.bas There are two methods that you need to override in a derived class: * Constructor - Define the parameters required for the operator. You only need to specify the arguments specific to your operator. - Use ``@apply_defaults`` decorator function to fill unspecified arguments with ``default_args``. You can specify the ``default_args`` - in the dag file. See :ref:`Default args ` for more details. + You can specify the ``default_args`` in the dag file. See :ref:`Default args ` for more details. * Execute - The code to execute when the runner calls the operator. The method contains the airflow context as a parameter that can be used to read config values. @@ -39,11 +38,9 @@ Let's implement an example ``HelloOperator`` in a new file ``hello_operator.py`` .. code-block:: python from airflow.models.baseoperator import BaseOperator - from airflow.utils.decorators import apply_defaults class HelloOperator(BaseOperator): - @apply_defaults def __init__( self, name: str, @@ -107,7 +104,6 @@ Let's extend our previous example to fetch name from MySQL: class HelloDBOperator(BaseOperator): - @apply_defaults def __init__( self, name: str, @@ -162,7 +158,6 @@ the operator. template_fields = ['name'] - @apply_defaults def __init__( self, name: str, @@ -198,7 +193,6 @@ with actual value. Note that Jinja substitutes the operator attributes and not t template_fields = ['guest_name'] template_ext = ['.sql'] - @apply_defaults def __init__( self, name: str, @@ -217,7 +211,6 @@ from template field renders in Web UI. For example: template_fields = ['request_body'] template_fields_renderers = {'request_body': 'json'} - @apply_defaults def __init__( self, request_body: str, diff --git a/docs/apache-airflow/howto/define_extra_link.rst b/docs/apache-airflow/howto/define_extra_link.rst index 374f69e161f70..cf7aba8b5f013 100644 --- a/docs/apache-airflow/howto/define_extra_link.rst +++ b/docs/apache-airflow/howto/define_extra_link.rst @@ -33,7 +33,6 @@ The following code shows how to add extra links to an operator via Plugins: from airflow.models.baseoperator import BaseOperator, BaseOperatorLink from airflow.plugins_manager import AirflowPlugin - from airflow.utils.decorators import apply_defaults class GoogleLink(BaseOperatorLink): @@ -48,7 +47,6 @@ The following code shows how to add extra links to an operator via Plugins: GoogleLink(), ) - @apply_defaults def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/docs/apache-airflow/img/dag_doc.png b/docs/apache-airflow/img/dag_doc.png new file mode 100644 index 0000000000000..22bc78c1e7473 Binary files /dev/null and b/docs/apache-airflow/img/dag_doc.png differ diff --git a/docs/apache-airflow/img/task_doc.png b/docs/apache-airflow/img/task_doc.png new file mode 100644 index 0000000000000..f31c99adc0033 Binary files /dev/null and b/docs/apache-airflow/img/task_doc.png differ diff --git a/docs/apache-airflow/index.rst b/docs/apache-airflow/index.rst index 4c0b773ff94b5..4588ca80290c2 100644 --- a/docs/apache-airflow/index.rst +++ b/docs/apache-airflow/index.rst @@ -113,7 +113,7 @@ unit of work and continuity. CLI Macros Python API - Experimental REST API Stable REST API + deprecated-rest-api-ref Configurations Extra packages diff --git a/docs/apache-airflow/installation.rst b/docs/apache-airflow/installation.rst index f410cdcfa5755..403a7e64940cb 100644 --- a/docs/apache-airflow/installation.rst +++ b/docs/apache-airflow/installation.rst @@ -157,7 +157,7 @@ not work or will produce unusable Airflow installation. In order to have repeatable installation, starting from **Airflow 1.10.10** and updated in **Airflow 1.10.13** we also keep a set of "known-to-be-working" constraint files in the -``constraints-master``, ``constraints-2-0`` and ``constraints-1-10`` orphan branches and then we create tag +``constraints-master``, ``constraints-2-0`` orphan branches and then we create tag for each released version e.g. :subst-code:`constraints-|version|`. This way, when we keep a tested and working set of dependencies. Those "known-to-be-working" constraints are per major/minor Python version. You can use them as constraint @@ -172,7 +172,7 @@ You can create the URL to the file substituting the variables in the template be where: -- ``AIRFLOW_VERSION`` - Airflow version (e.g. :subst-code:`|version|`) or ``master``, ``2-0``, ``1-10`` for latest development version +- ``AIRFLOW_VERSION`` - Airflow version (e.g. :subst-code:`|version|`) or ``master``, ``2-0``, for latest development version - ``PYTHON_VERSION`` Python version e.g. ``3.8``, ``3.7`` There is also a no-providers constraint file, which contains just constraints required to install Airflow core. This allows diff --git a/docs/apache-airflow/integration.rst b/docs/apache-airflow/integration.rst index 24c9de095b8fa..0221d348219bf 100644 --- a/docs/apache-airflow/integration.rst +++ b/docs/apache-airflow/integration.rst @@ -33,4 +33,4 @@ Airflow has a mechanism that allows you to expand its functionality and integrat * :doc:`Web UI Authentication backends ` It also has integration with :doc:`Sentry ` service for error tracking. Other applications can also integrate using -the :doc:`REST API `. +the :doc:`REST API `. diff --git a/docs/apache-airflow/redirects.txt b/docs/apache-airflow/redirects.txt index ea7712723a182..acfd6940ef689 100644 --- a/docs/apache-airflow/redirects.txt +++ b/docs/apache-airflow/redirects.txt @@ -43,6 +43,7 @@ start.rst start/index.rst # References cli-ref.rst cli-and-env-variables-ref.rst _api/index.rst python-api-ref.rst +rest-api-ref.rst deprecated-rest-api-ref.rst # Concepts concepts.rst concepts/index.rst diff --git a/docs/apache-airflow/security/access-control.rst b/docs/apache-airflow/security/access-control.rst index 38f537e6e9734..26157256abd5b 100644 --- a/docs/apache-airflow/security/access-control.rst +++ b/docs/apache-airflow/security/access-control.rst @@ -168,6 +168,7 @@ Endpoint /pools/{pool_name} DELETE Pool.can_delete Op /pools/{pool_name} GET Pool.can_read Op /pools/{pool_name} PATCH Pool.can_edit Op +/providers GET Provider.can_read Op /dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances GET DAGs.can_read, DAG Runs.can_read, Task Instances.can_read Viewer /dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances/{task_id} GET DAGs.can_read, DAG Runs.can_read, Task Instances.can_read Viewer /dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances/{task_id}/links GET DAGs.can_read, DAG Runs.can_read, Task Instances.can_read Viewer diff --git a/docs/apache-airflow/security/secrets/index.rst b/docs/apache-airflow/security/secrets/index.rst index 5cb0594fb76b6..fd5bbd0ce4f4e 100644 --- a/docs/apache-airflow/security/secrets/index.rst +++ b/docs/apache-airflow/security/secrets/index.rst @@ -30,6 +30,69 @@ The following are particularly protected: .. toctree:: :maxdepth: 1 :glob: + :caption: Further reading: - fernet - secrets-backend/index + Encryption at rest + Using external Secret stores + +.. _security:mask-sensitive-values: + +Masking sensitive data +---------------------- + +Airflow will by default mask Connection passwords and sensitive Variables and keys from a Connection's +extra (JSON) field when they appear in Task logs, in the Variable and in the Rendered fields views of the UI. + +It does this by looking for the specific *value* appearing anywhere in your output. This means that if you +have a connection with a password of ``a``, then every instance of the letter a in your logs will be replaced +with ``***``. + +To disable masking you can setting :ref:`config:core__hide_sensitive_var_conn_fields` to false. + +The automatic masking is triggered by Connection or Variable access. This means that if you pass a sensitive +value via XCom or any other side-channel it will not be masked when printed in the downstream task. + +Sensitive field names +""""""""""""""""""""" + +When masking is enabled, Airflow will always mask the password field of every Connection that is accessed by a +task. + +It will also mask the value of a Variable, or the field of a Connection's extra JSON blob if the name contains +any words in ('password', 'secret', 'passwd', 'authorization', 'api_key', 'apikey', 'access_token'). This list +can also be extended: + +.. code-block:: ini + + [core] + sensitive_var_conn_names = comma,separated,sensitive,names + +Adding your own masks +""""""""""""""""""""" + +If you want to mask an additional secret that is already masked by one of the above methods, you can do it in +your DAG file or operator's ``execute`` function using the ``mask_secret`` function. For example: + +.. code-block:: python + + @task + def my_func(): + from airflow.utils.log.secrets_masker import mask_secret + mask_secret("custom_value") + + ... + +or + +.. code-block:: python + + + class MyOperator(BaseOperator): + + def execute(self, context): + from airflow.utils.log.secrets_masker import mask_secret + mask_secret("custom_value") + + ... + +The mask must be set before any log/output is produced to have any effect. diff --git a/docs/apache-airflow/security/webserver.rst b/docs/apache-airflow/security/webserver.rst index 249cc01e93690..5fb03c5230167 100644 --- a/docs/apache-airflow/security/webserver.rst +++ b/docs/apache-airflow/security/webserver.rst @@ -38,14 +38,8 @@ set the below: Sensitive Variable fields ------------------------- -By default, Airflow Value of a variable will be hidden if the key contains any words in -(‘password’, ‘secret’, ‘passwd’, ‘authorization’, ‘api_key’, ‘apikey’, ‘access_token’), but can be configured -to extend this list by using the following configurations option: - -.. code-block:: ini - - [admin] - hide_sensitive_variable_fields = comma_separated_sensitive_variable_fields_list +Variable values that are deemed "sensitive" based on the variable name will be masked in the UI automatically. +See :ref:`security:mask-sensitive-values` for more details. .. _web-authentication: @@ -76,7 +70,7 @@ user will have by default: AUTH_ROLE_PUBLIC = 'Admin' -Be sure to checkout :doc:`/rest-api-ref` for securing the API. +Be sure to checkout :doc:`/security/api` for securing the API. .. note:: diff --git a/docs/apache-airflow/tutorial.rst b/docs/apache-airflow/tutorial.rst index b7db57a4ba913..b06288f2d35c6 100644 --- a/docs/apache-airflow/tutorial.rst +++ b/docs/apache-airflow/tutorial.rst @@ -184,7 +184,9 @@ Adding DAG and Tasks documentation ---------------------------------- We can add documentation for DAG or each single task. DAG documentation only support markdown so far and task documentation support plain text, markdown, reStructuredText, -json, yaml. +json, yaml. The DAG documentation can be written as a doc string at the beginning of the DAG file (recommended) +or anywhere in the file. Below you can find some examples on how to implement task and DAG docs, +as well as screenshots: .. exampleinclude:: /../../airflow/example_dags/tutorial.py :language: python @@ -192,6 +194,9 @@ json, yaml. :start-after: [START documentation] :end-before: [END documentation] +.. image:: img/task_doc.png +.. image:: img/dag_doc.png + Setting up Dependencies ----------------------- We have tasks ``t1``, ``t2`` and ``t3`` that do not depend on each other. Here's a few ways diff --git a/docs/apache-airflow/ui.rst b/docs/apache-airflow/ui.rst index 879d4d793d464..728c2aed46ef8 100644 --- a/docs/apache-airflow/ui.rst +++ b/docs/apache-airflow/ui.rst @@ -86,15 +86,7 @@ Variable View The variable view allows you to list, create, edit or delete the key-value pair of a variable used during jobs. Value of a variable will be hidden if the key contains any words in ('password', 'secret', 'passwd', 'authorization', 'api_key', 'apikey', 'access_token') -by default, but can be configured to show in clear-text (by configuration option -``hide_sensitive_variable_fields``). - -Users can also extend this list by using the following configurations option: - -.. code-block:: ini - - [admin] - sensitive_variable_fields = comma_separated_sensitive_variable_fields_list +by default, but can be configured to show in clear-text. See :ref:`security:mask-sensitive-values`. ------------ diff --git a/docs/apache-airflow/usage-cli.rst b/docs/apache-airflow/usage-cli.rst index 34f7ae77aa861..a62be8bb860e4 100644 --- a/docs/apache-airflow/usage-cli.rst +++ b/docs/apache-airflow/usage-cli.rst @@ -26,19 +26,6 @@ This document is meant to give an overview of all common tasks while using the C .. _cli-remote: -Set Up connection to a remote Airflow instance ----------------------------------------------- - -For some functions the CLI can use :doc:`the REST API `. To configure the CLI to use the API -when available configure as follows: - -.. code-block:: ini - - [cli] - api_client = airflow.api.client.json_client - endpoint_url = http://: - - Set Up Bash/Zsh Completion -------------------------- diff --git a/docs/build_docs.py b/docs/build_docs.py index c0b9b81840387..6bc60b6dcbf0c 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -27,7 +27,7 @@ from airflow.utils.helpers import partition from docs.exts.docs_build import dev_index_generator, lint_checks # pylint: disable=no-name-in-module -from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, PROVIDER_INIT_FILE, TEXT_RED, TEXT_RESET +from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, PROVIDER_INIT_FILE from docs.exts.docs_build.docs_builder import ( # pylint: disable=no-name-in-module DOCS_DIR, AirflowDocsBuilder, @@ -45,6 +45,9 @@ display_spelling_error_summary, ) +TEXT_RED = '\033[31m' +TEXT_RESET = '\033[0m' + if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index 20aa77a43911a..5af4cf7876ffc 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -58,9 +58,8 @@ Those are the most common arguments that you use when you want to build a custom +------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_CONSTRAINTS_REFERENCE`` | | Reference (branch or tag) from GitHub | | | | where constraints file is taken from | -| | | It can be ``constraints-master`` but | -| | | can be ``constraints-1-10`` for 1.10.* | -| | | versions of ``constraints-2-0`` for | +| | | It can be ``constraints-master`` or | +| | | ``constraints-2-0`` for | | | | 2.0.* installation. In case of building | | | | specific version you want to point it | | | | to specific tag, for example | diff --git a/docs/exts/docroles.py b/docs/exts/docroles.py index a1ac0cb2ecbee..f12f493bd4ee6 100644 --- a/docs/exts/docroles.py +++ b/docs/exts/docroles.py @@ -32,6 +32,7 @@ def get_template_field(env, fullname): """ Gets template fields for specific operator class. + :param env: env config :param fullname: Full path to operator class. For example: ``airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductSetOperator`` :return: List of template field diff --git a/docs/exts/docs_build/code_utils.py b/docs/exts/docs_build/code_utils.py index adab5c256f9ff..0a772f458282c 100644 --- a/docs/exts/docs_build/code_utils.py +++ b/docs/exts/docs_build/code_utils.py @@ -27,12 +27,9 @@ AIRFLOW_DIR = os.path.join(ROOT_PROJECT_DIR, "airflow") ALL_PROVIDER_YAMLS = load_package_data() -AIRFLOW_SITE_DIR = os.environ.get('AIRFLOW_SITE_DIRECTORY') +AIRFLOW_SITE_DIR: str = os.environ.get('AIRFLOW_SITE_DIRECTORY') or '' PROCESS_TIMEOUT = 8 * 60 # 400 seconds -TEXT_RED = '\033[31m' -TEXT_RESET = '\033[0m' - CONSOLE_WIDTH = 180 @@ -81,11 +78,8 @@ def guess_lexer_for_filename(filename): def pretty_format_path(path: str, start: str) -> str: - """Formats the path by marking the important part in bold.""" - end = '\033[0m' - bold = '\033[1m' - + """Formats path nicely.""" relpath = os.path.relpath(path, start) if relpath == path: - return f"{bold}path{end}" - return f"{start}/{bold}{relpath}{end}" + return path + return f"{start}/{relpath}" diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py index 44cf5acd6826f..55ae49096829e 100644 --- a/docs/exts/docs_build/docs_builder.py +++ b/docs/exts/docs_build/docs_builder.py @@ -271,7 +271,7 @@ def build_sphinx_docs(self, verbose: bool) -> List[DocBuildError]: console.print(f"[blue]{self.package_name:60}:[/] [green]Finished docs building successfully[/]") return build_errors - def publish(self): + def publish(self, override_versioned: bool): """Copy documentation packages files to airflow-site repository.""" console.print(f"Publishing docs for {self.package_name}") output_dir = os.path.join(AIRFLOW_SITE_DIR, self._publish_dir) @@ -280,14 +280,16 @@ def publish(self): console.print(f"Copy directory: {pretty_source} => {pretty_target}") if os.path.exists(output_dir): if self.is_versioned: - console.print( - f"Skipping previously existing {output_dir}! " - f"Delete it manually if you want to regenerate it!" - ) - console.print() - return - else: - shutil.rmtree(output_dir) + if override_versioned: + console.print(f"Overriding previously existing {output_dir}! ") + else: + console.print( + f"Skipping previously existing {output_dir}! " + f"Delete it manually if you want to regenerate it!" + ) + console.print() + return + shutil.rmtree(output_dir) shutil.copytree(self._build_dir, output_dir) if self.is_versioned: with open(os.path.join(output_dir, "..", "stable.txt"), "w") as stable_file: diff --git a/docs/exts/docs_build/errors.py b/docs/exts/docs_build/errors.py index 3fe9f36d810b3..e334d0800e3c4 100644 --- a/docs/exts/docs_build/errors.py +++ b/docs/exts/docs_build/errors.py @@ -87,6 +87,7 @@ def parse_sphinx_warnings(warning_text: str, docs_dir: str) -> List[DocBuildErro Parses warnings from Sphinx. :param warning_text: warning to parse + :param docs_dir: documentation directory :return: list of DocBuildErrors. """ sphinx_build_errors = [] diff --git a/docs/exts/docs_build/lint_checks.py b/docs/exts/docs_build/lint_checks.py index 155b8f5b4d0ad..5fd996d544b4c 100644 --- a/docs/exts/docs_build/lint_checks.py +++ b/docs/exts/docs_build/lint_checks.py @@ -195,7 +195,7 @@ def _extract_file_content(file_path: str, message: Optional[str], pattern: str, def filter_file_list_by_pattern(file_paths: Iterable[str], pattern: str) -> List[str]: """ - Filters file list to those tha content matches the pattern + Filters file list to those that content matches the pattern :param file_paths: file paths to check :param pattern: pattern to match :return: list of files matching the pattern diff --git a/docs/exts/docs_build/spelling_checks.py b/docs/exts/docs_build/spelling_checks.py index 24ce3f170584f..4da70c11894ba 100644 --- a/docs/exts/docs_build/spelling_checks.py +++ b/docs/exts/docs_build/spelling_checks.py @@ -85,6 +85,7 @@ def parse_spelling_warnings(warning_text: str, docs_dir: str) -> List[SpellingEr Parses warnings from Sphinx. :param warning_text: warning to parse + :param docs_dir: documentation directory :return: list of SpellingError. """ sphinx_spelling_errors = [] diff --git a/docs/exts/substitution_extensions.py b/docs/exts/substitution_extensions.py index bbfe6537cf68a..1cf3c6d6a28bf 100644 --- a/docs/exts/substitution_extensions.py +++ b/docs/exts/substitution_extensions.py @@ -51,7 +51,7 @@ def run(self) -> list: class SubstitutionCodeBlockTransform(SphinxTransform): - """Substitue ``|variables|`` in code and code-block nodes""" + """Substitute ``|variables|`` in code and code-block nodes""" # Run before we highlight the code! default_priority = HighlightLanguageTransform.default_priority - 1 diff --git a/docs/helm-chart/parameters-ref.rst b/docs/helm-chart/parameters-ref.rst index f6a5097d6c6d0..9370bebe328bb 100644 --- a/docs/helm-chart/parameters-ref.rst +++ b/docs/helm-chart/parameters-ref.rst @@ -83,64 +83,67 @@ The following tables lists the configurable parameters of the Airflow chart and - ``1`` * - ``defaultAirflowRepository`` - Fallback docker repository to pull airflow image from - - ``1`` + - ``apache/airflow`` * - ``defaultAirflowTag`` - Fallback docker image tag to deploy - - ``1`` + - ``2.0.2`` + * - ``airflowVersion`` + - Airflow version (Used to make some decisions based on Airflow Version being deployed) + - ``2.0.2`` * - ``images.airflow.repository`` - Docker repository to pull image from. Update this to deploy a custom image - - ``1`` + - ``~`` * - ``images.airflow.tag`` - Docker image tag to pull image from. Update this to deploy a new custom image tag - - ``1`` + - ``~`` * - ``images.airflow.pullPolicy`` - PullPolicy for airflow image - - ``1`` + - ``IfNotPresent`` * - ``images.flower.repository`` - Docker repository to pull image from. Update this to deploy a custom image - - ``1`` + - ``~`` * - ``images.flower.tag`` - Docker image tag to pull image from. Update this to deploy a new custom image tag - - ``1`` + - ``~`` * - ``images.flower.pullPolicy`` - PullPolicy for flower image - - ``1`` + - ``IfNotPresent`` * - ``images.statsd.repository`` - Docker repository to pull image from. Update this to deploy a custom image - - ``1`` + - ``apache/airflow`` * - ``images.statsd.tag`` - Docker image tag to pull image from. Update this to deploy a new custom image tag - - ``1`` + - ``airflow-statsd-exporter-2021.04.28-v0.17.0`` * - ``images.statsd.pullPolicy`` - PullPolicy for statsd-exporter image - - ``1`` + - ``IfNotPresent`` * - ``images.redis.repository`` - Docker repository to pull image from. Update this to deploy a custom image - - ``1`` + - ``redis`` * - ``images.redis.tag`` - Docker image tag to pull image from. Update this to deploy a new custom image tag - - ``1`` + - ``6-buster`` * - ``images.redis.pullPolicy`` - PullPolicy for redis image - - ``1`` + - ``IfNotPresent`` * - ``images.pgbouncer.repository`` - Docker repository to pull image from. Update this to deploy a custom image - - ``1`` + - ``apache/airflow`` * - ``images.pgbouncer.tag`` - Docker image tag to pull image from. Update this to deploy a new custom image tag - - ``1`` + - ``airflow-pgbouncer-2021.04.28-1.14.0`` * - ``images.pgbouncer.pullPolicy`` - PullPolicy for PgBouncer image - - ``1`` + - ``IfNotPresent`` * - ``images.pgbouncerExporter.repository`` - Docker repository to pull image from. Update this to deploy a custom image - - ``1`` + - ``apache/airflow`` * - ``images.pgbouncerExporter.tag`` - Docker image tag to pull image from. Update this to deploy a new custom image tag - - ``1`` + - ``airflow-pgbouncer-exporter-2021.04.28-0.5.0`` * - ``images.pgbouncerExporter.pullPolicy`` - PullPolicy for ``pgbouncer-exporter`` image - - ``1`` + - ``IfNotPresent`` * - ``env`` - Environment variables key/values to mount into Airflow pods (deprecated, prefer using ``extraEnv``) - ``1`` diff --git a/docs/integration-logos/postgress/Postgress.png b/docs/integration-logos/postgres/Postgres.png similarity index 100% rename from docs/integration-logos/postgress/Postgress.png rename to docs/integration-logos/postgres/Postgres.png diff --git a/docs/publish_docs.py b/docs/publish_docs.py index 92476c3f46ec3..0be5f61cd093d 100755 --- a/docs/publish_docs.py +++ b/docs/publish_docs.py @@ -65,6 +65,12 @@ def _get_parser(): parser.add_argument( '--disable-checks', dest='disable_checks', action='store_true', help='Disables extra checks' ) + parser.add_argument( + '--override-versioned', + dest='override_versioned', + action='store_true', + help='Overrides versioned directories', + ) parser.add_argument( "--package-filter", action="append", @@ -90,7 +96,7 @@ def main(): print() for package_name in current_packages: builder = AirflowDocsBuilder(package_name=package_name, for_production=True) - builder.publish() + builder.publish(override_versioned=args.override_versioned) main() diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 464db0cb14e4d..6c6ab381c3101 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1039,6 +1039,7 @@ nodeName nodeSelector nonterminal noqa +nosasl notificationChannels npm ntlm diff --git a/kubernetes_tests/test_base.py b/kubernetes_tests/test_base.py new file mode 100644 index 0000000000000..5c2ef6ad2be12 --- /dev/null +++ b/kubernetes_tests/test_base.py @@ -0,0 +1,209 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import re +import subprocess +import time +import unittest +from datetime import datetime +from subprocess import check_call, check_output + +import requests +import requests.exceptions +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +CLUSTER_FORWARDED_PORT = os.environ.get('CLUSTER_FORWARDED_PORT') or "8080" +KUBERNETES_HOST_PORT = (os.environ.get('CLUSTER_HOST') or "localhost") + ":" + CLUSTER_FORWARDED_PORT +EXECUTOR = os.environ.get("EXECUTOR") + +print() +print(f"Cluster host/port used: ${KUBERNETES_HOST_PORT}") +print(f"Executor: {EXECUTOR}") +print() + + +class TestBase(unittest.TestCase): + @staticmethod + def _describe_resources(namespace: str): + print("=" * 80) + print(f"Describe resources for namespace {namespace}") + print(f"Datetime: {datetime.utcnow()}") + print("=" * 80) + print("Describing pods") + print("-" * 80) + subprocess.call(["kubectl", "describe", "pod", "--namespace", namespace]) + print("=" * 80) + print("Describing persistent volumes") + print("-" * 80) + subprocess.call(["kubectl", "describe", "pv", "--namespace", namespace]) + print("=" * 80) + print("Describing persistent volume claims") + print("-" * 80) + subprocess.call(["kubectl", "describe", "pvc", "--namespace", namespace]) + print("=" * 80) + + @staticmethod + def _num_pods_in_namespace(namespace): + air_pod = check_output(['kubectl', 'get', 'pods', '-n', namespace]).decode() + air_pod = air_pod.split('\n') + names = [re.compile(r'\s+').split(x)[0] for x in air_pod if 'airflow' in x] + return len(names) + + @staticmethod + def _delete_airflow_pod(name=''): + suffix = '-' + name if name else '' + air_pod = check_output(['kubectl', 'get', 'pods']).decode() + air_pod = air_pod.split('\n') + names = [re.compile(r'\s+').split(x)[0] for x in air_pod if 'airflow' + suffix in x] + if names: + check_call(['kubectl', 'delete', 'pod', names[0]]) + + def _get_session_with_retries(self): + session = requests.Session() + session.auth = ('admin', 'admin') + retries = Retry(total=3, backoff_factor=1) + session.mount('http://', HTTPAdapter(max_retries=retries)) + session.mount('https://', HTTPAdapter(max_retries=retries)) + return session + + def _ensure_airflow_webserver_is_healthy(self): + response = self.session.get( + f"http://{KUBERNETES_HOST_PORT}/health", + timeout=1, + ) + + assert response.status_code == 200 + + def setUp(self): + self.host = KUBERNETES_HOST_PORT + self.session = self._get_session_with_retries() + self._ensure_airflow_webserver_is_healthy() + + def tearDown(self): + self.session.close() + + def monitor_task(self, host, dag_run_id, dag_id, task_id, expected_final_state, timeout): + tries = 0 + state = '' + max_tries = max(int(timeout / 5), 1) + # Wait some time for the operator to complete + while tries < max_tries: + time.sleep(5) + # Check task state + try: + get_string = ( + f'http://{host}/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances/{task_id}' + ) + print(f"Calling [monitor_task]#1 {get_string}") + result = self.session.get(get_string) + if result.status_code == 404: + check_call(["echo", "api returned 404."]) + tries += 1 + continue + assert result.status_code == 200, "Could not get the status" + result_json = result.json() + print(f"Received [monitor_task]#2: {result_json}") + state = result_json['state'] + print(f"Attempt {tries}: Current state of operator is {state}") + + if state == expected_final_state: + break + self._describe_resources(namespace="airflow") + self._describe_resources(namespace="default") + tries += 1 + except requests.exceptions.ConnectionError as e: + check_call(["echo", f"api call failed. trying again. error {e}"]) + if state != expected_final_state: + print(f"The expected state is wrong {state} != {expected_final_state} (expected)!") + assert state == expected_final_state + + def ensure_dag_expected_state(self, host, execution_date, dag_id, expected_final_state, timeout): + tries = 0 + state = '' + max_tries = max(int(timeout / 5), 1) + # Wait some time for the operator to complete + while tries < max_tries: + time.sleep(5) + get_string = f'http://{host}/api/v1/dags/{dag_id}/dagRuns' + print(f"Calling {get_string}") + # Get all dagruns + result = self.session.get(get_string) + assert result.status_code == 200, "Could not get the status" + result_json = result.json() + print(f"Received: {result}") + state = None + for dag_run in result_json['dag_runs']: + if dag_run['execution_date'] == execution_date: + state = dag_run['state'] + check_call(["echo", f"Attempt {tries}: Current state of dag is {state}"]) + print(f"Attempt {tries}: Current state of dag is {state}") + + if state == expected_final_state: + break + self._describe_resources("airflow") + self._describe_resources("default") + tries += 1 + assert state == expected_final_state + + # Maybe check if we can retrieve the logs, but then we need to extend the API + + def start_dag(self, dag_id, host): + patch_string = f'http://{host}/api/v1/dags/{dag_id}' + print(f"Calling [start_dag]#1 {patch_string}") + result = self.session.patch(patch_string, json={'is_paused': False}) + try: + result_json = result.json() + except ValueError: + result_json = str(result) + print(f"Received [start_dag]#1 {result_json}") + assert result.status_code == 200, f"Could not enable DAG: {result_json}" + post_string = f'http://{host}/api/v1/dags/{dag_id}/dagRuns' + print(f"Calling [start_dag]#2 {post_string}") + # Trigger a new dagrun + result = self.session.post(post_string, json={}) + try: + result_json = result.json() + except ValueError: + result_json = str(result) + print(f"Received [start_dag]#2 {result_json}") + assert result.status_code == 200, f"Could not trigger a DAG-run: {result_json}" + + time.sleep(1) + + get_string = f'http://{host}/api/v1/dags/{dag_id}/dagRuns' + print(f"Calling [start_dag]#3 {get_string}") + result = self.session.get(get_string) + assert result.status_code == 200, f"Could not get DAGRuns: {result.json()}" + result_json = result.json() + print(f"Received: [start_dag]#3 {result_json}") + return result_json + + def start_job_in_kubernetes(self, dag_id, host): + result_json = self.start_dag(dag_id=dag_id, host=host) + dag_runs = result_json['dag_runs'] + assert len(dag_runs) > 0 + execution_date = None + dag_run_id = None + for dag_run in dag_runs: + if dag_run['dag_id'] == dag_id: + execution_date = dag_run['execution_date'] + dag_run_id = dag_run['dag_run_id'] + break + assert execution_date is not None, f"No execution_date can be found for the dag with {dag_id}" + return dag_run_id, execution_date diff --git a/kubernetes_tests/test_kubernetes_executor.py b/kubernetes_tests/test_kubernetes_executor.py index 2933547eda37b..1ca3848fe5bb9 100644 --- a/kubernetes_tests/test_kubernetes_executor.py +++ b/kubernetes_tests/test_kubernetes_executor.py @@ -14,201 +14,25 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import os -import re -import subprocess -import time -import unittest -from datetime import datetime -from subprocess import check_call, check_output - -import requests -import requests.exceptions -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry - -CLUSTER_FORWARDED_PORT = os.environ.get('CLUSTER_FORWARDED_PORT') or "8080" -KUBERNETES_HOST_PORT = (os.environ.get('CLUSTER_HOST') or "localhost") + ":" + CLUSTER_FORWARDED_PORT - -print() -print(f"Cluster host/port used: ${KUBERNETES_HOST_PORT}") -print() - - -class TestKubernetesExecutor(unittest.TestCase): - @staticmethod - def _describe_resources(namespace: str): - print("=" * 80) - print(f"Describe resources for namespace {namespace}") - print(f"Datetime: {datetime.utcnow()}") - print("=" * 80) - print("Describing pods") - print("-" * 80) - subprocess.call(["kubectl", "describe", "pod", "--namespace", namespace]) - print("=" * 80) - print("Describing persistent volumes") - print("-" * 80) - subprocess.call(["kubectl", "describe", "pv", "--namespace", namespace]) - print("=" * 80) - print("Describing persistent volume claims") - print("-" * 80) - subprocess.call(["kubectl", "describe", "pvc", "--namespace", namespace]) - print("=" * 80) - - @staticmethod - def _num_pods_in_namespace(namespace): - air_pod = check_output(['kubectl', 'get', 'pods', '-n', namespace]).decode() - air_pod = air_pod.split('\n') - names = [re.compile(r'\s+').split(x)[0] for x in air_pod if 'airflow' in x] - return len(names) - - @staticmethod - def _delete_airflow_pod(name=''): - suffix = '-' + name if name else '' - air_pod = check_output(['kubectl', 'get', 'pods']).decode() - air_pod = air_pod.split('\n') - names = [re.compile(r'\s+').split(x)[0] for x in air_pod if 'airflow' + suffix in x] - if names: - check_call(['kubectl', 'delete', 'pod', names[0]]) - - def _get_session_with_retries(self): - session = requests.Session() - retries = Retry(total=3, backoff_factor=1) - session.mount('http://', HTTPAdapter(max_retries=retries)) - session.mount('https://', HTTPAdapter(max_retries=retries)) - return session - - def _ensure_airflow_webserver_is_healthy(self): - response = self.session.get( - f"http://{KUBERNETES_HOST_PORT}/health", - timeout=1, - ) - - assert response.status_code == 200 - - def setUp(self): - self.session = self._get_session_with_retries() - self._ensure_airflow_webserver_is_healthy() - def tearDown(self): - self.session.close() - - def monitor_task(self, host, execution_date, dag_id, task_id, expected_final_state, timeout): - tries = 0 - state = '' - max_tries = max(int(timeout / 5), 1) - # Wait some time for the operator to complete - while tries < max_tries: - time.sleep(5) - # Trigger a new dagrun - try: - get_string = ( - f'http://{host}/api/experimental/dags/{dag_id}/' - f'dag_runs/{execution_date}/tasks/{task_id}' - ) - print(f"Calling [monitor_task]#1 {get_string}") - result = self.session.get(get_string) - if result.status_code == 404: - check_call(["echo", "api returned 404."]) - tries += 1 - continue - assert result.status_code == 200, "Could not get the status" - result_json = result.json() - print(f"Received [monitor_task]#2: {result_json}") - state = result_json['state'] - print(f"Attempt {tries}: Current state of operator is {state}") - - if state == expected_final_state: - break - self._describe_resources(namespace="airflow") - self._describe_resources(namespace="default") - tries += 1 - except requests.exceptions.ConnectionError as e: - check_call(["echo", f"api call failed. trying again. error {e}"]) - if state != expected_final_state: - print(f"The expected state is wrong {state} != {expected_final_state} (expected)!") - assert state == expected_final_state - - def ensure_dag_expected_state(self, host, execution_date, dag_id, expected_final_state, timeout): - tries = 0 - state = '' - max_tries = max(int(timeout / 5), 1) - # Wait some time for the operator to complete - while tries < max_tries: - time.sleep(5) - get_string = f'http://{host}/api/experimental/dags/{dag_id}/dag_runs/{execution_date}' - print(f"Calling {get_string}") - # Trigger a new dagrun - result = self.session.get(get_string) - assert result.status_code == 200, "Could not get the status" - result_json = result.json() - print(f"Received: {result}") - state = result_json['state'] - check_call(["echo", f"Attempt {tries}: Current state of dag is {state}"]) - print(f"Attempt {tries}: Current state of dag is {state}") - - if state == expected_final_state: - break - self._describe_resources("airflow") - self._describe_resources("default") - tries += 1 - assert state == expected_final_state - - # Maybe check if we can retrieve the logs, but then we need to extend the API - - def start_dag(self, dag_id, host): - get_string = f'http://{host}/api/experimental/dags/{dag_id}/paused/false' - print(f"Calling [start_dag]#1 {get_string}") - result = self.session.get(get_string) - try: - result_json = result.json() - except ValueError: - result_json = str(result) - print(f"Received [start_dag]#1 {result_json}") - assert result.status_code == 200, f"Could not enable DAG: {result_json}" - post_string = f'http://{host}/api/experimental/dags/{dag_id}/dag_runs' - print(f"Calling [start_dag]#2 {post_string}") - # Trigger a new dagrun - result = self.session.post(post_string, json={}) - try: - result_json = result.json() - except ValueError: - result_json = str(result) - print(f"Received [start_dag]#2 {result_json}") - assert result.status_code == 200, f"Could not trigger a DAG-run: {result_json}" +import time - time.sleep(1) +import pytest - get_string = f'http://{host}/api/experimental/latest_runs' - print(f"Calling [start_dag]#3 {get_string}") - result = self.session.get(get_string) - assert result.status_code == 200, f"Could not get the latest DAG-run: {result.json()}" - result_json = result.json() - print(f"Received: [start_dag]#3 {result_json}") - return result_json +from kubernetes_tests.test_base import EXECUTOR, TestBase - def start_job_in_kubernetes(self, dag_id, host): - result_json = self.start_dag(dag_id=dag_id, host=host) - assert len(result_json['items']) > 0 - execution_date = None - for dag_run in result_json['items']: - if dag_run['dag_id'] == dag_id: - execution_date = dag_run['execution_date'] - break - assert execution_date is not None, f"No execution_date can be found for the dag with {dag_id}" - return execution_date +@pytest.mark.skipif(EXECUTOR != 'KubernetesExecutor', reason="Only runs on KubernetesExecutor") +class TestKubernetesExecutor(TestBase): def test_integration_run_dag(self): - host = KUBERNETES_HOST_PORT dag_id = 'example_kubernetes_executor_config' - - execution_date = self.start_job_in_kubernetes(dag_id, host) - print(f"Found the job with execution date {execution_date}") + dag_run_id, execution_date = self.start_job_in_kubernetes(dag_id, self.host) + print(f"Found the job with execution_date {execution_date}") # Wait some time for the operator to complete self.monitor_task( - host=host, - execution_date=execution_date, + host=self.host, + dag_run_id=dag_run_id, dag_id=dag_id, task_id='start_task', expected_final_state='success', @@ -216,7 +40,7 @@ def test_integration_run_dag(self): ) self.ensure_dag_expected_state( - host=host, + host=self.host, execution_date=execution_date, dag_id=dag_id, expected_final_state='success', @@ -224,10 +48,9 @@ def test_integration_run_dag(self): ) def test_integration_run_dag_with_scheduler_failure(self): - host = KUBERNETES_HOST_PORT dag_id = 'example_kubernetes_executor_config' - execution_date = self.start_job_in_kubernetes(dag_id, host) + dag_run_id, execution_date = self.start_job_in_kubernetes(dag_id, self.host) self._delete_airflow_pod("scheduler") @@ -235,8 +58,8 @@ def test_integration_run_dag_with_scheduler_failure(self): # Wait some time for the operator to complete self.monitor_task( - host=host, - execution_date=execution_date, + host=self.host, + dag_run_id=dag_run_id, dag_id=dag_id, task_id='start_task', expected_final_state='success', @@ -244,8 +67,8 @@ def test_integration_run_dag_with_scheduler_failure(self): ) self.monitor_task( - host=host, - execution_date=execution_date, + host=self.host, + dag_run_id=dag_run_id, dag_id=dag_id, task_id='other_namespace_task', expected_final_state='success', @@ -253,7 +76,7 @@ def test_integration_run_dag_with_scheduler_failure(self): ) self.ensure_dag_expected_state( - host=host, + host=self.host, execution_date=execution_date, dag_id=dag_id, expected_final_state='success', diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes_tests/test_kubernetes_pod_operator.py index 55d03cb298fce..d518da6377ef6 100644 --- a/kubernetes_tests/test_kubernetes_pod_operator.py +++ b/kubernetes_tests/test_kubernetes_pod_operator.py @@ -851,7 +851,6 @@ def test_pod_template_file( 'metadata': { 'annotations': {}, 'labels': { - 'airflow_version': '2.1.0.dev0', 'dag_id': 'dag', 'execution_date': mock.ANY, 'kubernetes_pod_operator': 'True', @@ -895,6 +894,9 @@ def test_pod_template_file( 'volumes': [{'emptyDir': {}, 'name': 'xcom'}], }, } + version = actual_pod['metadata']['labels']['airflow_version'] + assert version.startswith(airflow_version) + del actual_pod['metadata']['labels']['airflow_version'] assert expected_dict == actual_pod @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_launcher.PodLauncher.start_pod") diff --git a/kubernetes_tests/test_other_executors.py b/kubernetes_tests/test_other_executors.py new file mode 100644 index 0000000000000..9c92dbca4aae7 --- /dev/null +++ b/kubernetes_tests/test_other_executors.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import time + +import pytest + +from kubernetes_tests.test_base import EXECUTOR, TestBase + + +# These tests are here because only KubernetesExecutor can run the tests in +# test_kubernetes_executor.py +# Also, the skipping is necessary as there's no gain in running these tests in KubernetesExecutor +@pytest.mark.skipif(EXECUTOR == 'KubernetesExecutor', reason="Does not run on KubernetesExecutor") +class TestCeleryAndLocalExecutor(TestBase): + def test_integration_run_dag(self): + dag_id = 'example_bash_operator' + dag_run_id, execution_date = self.start_job_in_kubernetes(dag_id, self.host) + print(f"Found the job with execution_date {execution_date}") + + # Wait some time for the operator to complete + self.monitor_task( + host=self.host, + dag_run_id=dag_run_id, + dag_id=dag_id, + task_id='run_after_loop', + expected_final_state='success', + timeout=40, + ) + + self.ensure_dag_expected_state( + host=self.host, + execution_date=execution_date, + dag_id=dag_id, + expected_final_state='success', + timeout=60, + ) + + def test_integration_run_dag_with_scheduler_failure(self): + dag_id = 'example_xcom' + + dag_run_id, execution_date = self.start_job_in_kubernetes(dag_id, self.host) + + self._delete_airflow_pod("scheduler") + + time.sleep(10) # give time for pod to restart + + # Wait some time for the operator to complete + self.monitor_task( + host=self.host, + dag_run_id=dag_run_id, + dag_id=dag_id, + task_id='push', + expected_final_state='success', + timeout=40, # This should fail fast if failing + ) + + self.monitor_task( + host=self.host, + dag_run_id=dag_run_id, + dag_id=dag_id, + task_id='puller', + expected_final_state='success', + timeout=40, + ) + + self.ensure_dag_expected_state( + host=self.host, + execution_date=execution_date, + dag_id=dag_id, + expected_final_state='success', + timeout=60, + ) + + assert self._num_pods_in_namespace('test-namespace') == 0, "failed to delete pods in other namespace" diff --git a/pylintrc b/pylintrc index b3be01db95cf0..a44a5f9699ea1 100644 --- a/pylintrc +++ b/pylintrc @@ -67,7 +67,7 @@ confidence= # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if +# disable everything first and then re-enable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes diff --git a/pylintrc-tests b/pylintrc-tests index da6c76b81cab0..841fe0b381fea 100644 --- a/pylintrc-tests +++ b/pylintrc-tests @@ -67,7 +67,7 @@ confidence= # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if +# disable everything first and then re-enable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes diff --git a/scripts/ci/build_airflow/ci_build_airflow_package.sh b/scripts/ci/build_airflow/ci_build_airflow_packages.sh similarity index 88% rename from scripts/ci/build_airflow/ci_build_airflow_package.sh rename to scripts/ci/build_airflow/ci_build_airflow_packages.sh index a3706ad9f53f6..e6c405c6c36d2 100755 --- a/scripts/ci/build_airflow/ci_build_airflow_package.sh +++ b/scripts/ci/build_airflow/ci_build_airflow_packages.sh @@ -18,6 +18,6 @@ # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" -build_airflow_packages::build_airflow_packages - -cd "${AIRFLOW_SOURCES}/dist" || exit 1 +build_images::prepare_ci_build +build_images::rebuild_ci_image_if_needed_with_group +runs::run_prepare_airflow_packages diff --git a/scripts/ci/constraints/ci_branch_constraints.sh b/scripts/ci/constraints/ci_branch_constraints.sh index 1f733d44b0b19..b8041b169b07b 100755 --- a/scripts/ci/constraints/ci_branch_constraints.sh +++ b/scripts/ci/constraints/ci_branch_constraints.sh @@ -22,8 +22,6 @@ if [[ ${GITHUB_REF} == 'refs/heads/main' ]]; then echo "::set-output name=branch::constraints-main" elif [[ ${GITHUB_REF} == 'refs/heads/master' ]]; then echo "::set-output name=branch::constraints-master" -elif [[ ${GITHUB_REF} == 'refs/heads/v1-10-test' ]]; then - echo "::set-output name=branch::constraints-1-10" elif [[ ${GITHUB_REF} == 'refs/heads/v2-0-test' ]]; then echo "::set-output name=branch::constraints-2-0" else diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index 95375897a8986..6e8a5c5297662 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -27,7 +27,6 @@ CI_TARGET_BRANCH COMMIT_SHA DB_RESET DEFAULT_CONSTRAINTS_BRANCH -DISABLE_RBAC ENABLED_INTEGRATIONS ENABLED_SYSTEMS GITHUB_ACTIONS @@ -62,3 +61,4 @@ VERBOSE VERBOSE_COMMANDS VERSION_SUFFIX_FOR_PYPI VERSION_SUFFIX_FOR_SVN +WHEEL_VERSION diff --git a/scripts/ci/images/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh index ed2d524e2a19c..12e158b44c7c3 100755 --- a/scripts/ci/images/ci_build_dockerhub.sh +++ b/scripts/ci/images/ci_build_dockerhub.sh @@ -54,10 +54,10 @@ if [[ ! "${DOCKER_TAG}" =~ ^[0-9].* ]]; then echo # All the packages: Airflow and providers will have a "dev" version suffix in the imaage that # is built from non-release tag. If this is not set, then building images from locally build - # packages fails, because the packages with non-dev version are skipped (as they are alredy released) - export VERSION_SUFFIX_FOR_PYPI="dev" - export VERSION_SUFFIX_FOR_SVN="dev" - # Only build and push CI image for the nightly-master, v1-10-test and v2-0-test branches + # packages fails, because the packages with non-dev version are skipped (as they are already released) + export VERSION_SUFFIX_FOR_PYPI=".dev0" + export VERSION_SUFFIX_FOR_SVN=".dev0" + # Only build and push CI image for the nightly-master, v2-0-test branches # for tagged releases we build everything from PyPI, so we do not need CI images # For development images, we have to build all packages from current sources because we want to produce # `Latest and greatest` image from those branches. We need to build and push CI image as well as PROD diff --git a/scripts/ci/libraries/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh index 04e25e811a8ae..24be4749a8ee9 100755 --- a/scripts/ci/libraries/_all_libs.sh +++ b/scripts/ci/libraries/_all_libs.sh @@ -36,8 +36,6 @@ readonly SCRIPTS_CI_DIR . "${LIBRARIES_DIR}"/_repeats.sh # shellcheck source=scripts/ci/libraries/_sanity_checks.sh . "${LIBRARIES_DIR}"/_sanity_checks.sh -# shellcheck source=scripts/ci/libraries/_build_airflow_packages.sh -. "${LIBRARIES_DIR}"/_build_airflow_packages.sh # shellcheck source=scripts/ci/libraries/_build_images.sh . "${LIBRARIES_DIR}"/_build_images.sh # shellcheck source=scripts/ci/libraries/_kind.sh diff --git a/scripts/ci/libraries/_build_airflow_packages.sh b/scripts/ci/libraries/_build_airflow_packages.sh deleted file mode 100644 index 46429308c4060..0000000000000 --- a/scripts/ci/libraries/_build_airflow_packages.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Build airflow packages -function build_airflow_packages::build_airflow_packages() { - start_end::group_start "Build airflow packages ${PACKAGE_FORMAT}" - rm -rf -- *egg-info* - rm -rf -- build - - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" - - local packages=() - - if [[ ${PACKAGE_FORMAT} == "wheel" || ${PACKAGE_FORMAT} == "both" ]] ; then - packages+=("bdist_wheel") - fi - if [[ ${PACKAGE_FORMAT} == "sdist" || ${PACKAGE_FORMAT} == "both" ]] ; then - packages+=("sdist") - fi - - # Prepare airflow's wheel - PYTHONUNBUFFERED=1 python setup.py compile_assets "${packages[@]}" - - # clean-up - rm -rf -- *egg-info* - rm -rf -- build - - echo - echo "${COLOR_GREEN}Airflow package prepared in format: ${PACKAGE_FORMAT}${COLOR_RESET}" - echo - start_end::group_end -} diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 328d9d50155d0..51763df6dbcdf 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -36,23 +36,7 @@ function build_images::add_build_args_for_remote_install() { "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE}" ) else - if [[ ${AIRFLOW_VERSION} =~ [^0-9]*1[^0-9]*10[^0-9]([0-9]*) ]]; then - # All types of references/versions match this regexp for 1.10 series - # for example v1_10_test, 1.10.10, 1.10.9 etc. ${BASH_REMATCH[1]} matches last - # minor digit of version and it's length is 0 for v1_10_test, 1 for 1.10.9 and 2 for 1.10.10+ - AIRFLOW_MINOR_VERSION_NUMBER=${BASH_REMATCH[1]} - if [[ ${#AIRFLOW_MINOR_VERSION_NUMBER} == "0" ]]; then - # For v1_10_* branches use constraints-1-10 branch - EXTRA_DOCKER_PROD_BUILD_FLAGS+=( - "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=constraints-1-10" - ) - else - EXTRA_DOCKER_PROD_BUILD_FLAGS+=( - # For specified minor version of 1.10 or v1 branch use specific reference constraints - "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}" - ) - fi - elif [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then + if [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then EXTRA_DOCKER_PROD_BUILD_FLAGS+=( # For specified minor version of 2.0 or v2 branch use specific reference constraints "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}" @@ -70,14 +54,10 @@ function build_images::add_build_args_for_remote_install() { ) fi # Depending on the version built, we choose the right branch for preloading the packages from - # If we run build for v1-10-test builds we should choose v1-10-test, for v2-0-test we choose v2-0-test + # For v2-0-test we choose v2-0-test # all other builds when you choose a specific version (1.0 or 2.0 series) should choose stable branch # to preload. For all other builds we use the default branch defined in _initialization.sh - if [[ ${AIRFLOW_VERSION} == 'v1-10-test' ]]; then - AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v1-10-test" - elif [[ ${AIRFLOW_VERSION} =~ v?1.* ]]; then - AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v1-10-stable" - elif [[ ${AIRFLOW_VERSION} == 'v2-0-test' ]]; then + if [[ ${AIRFLOW_VERSION} == 'v2-0-test' ]]; then AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v2-0-test" elif [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v2-0-stable" @@ -134,7 +114,7 @@ function build_images::confirm_via_terminal() { RES=$? } -# Confirms if hte image should be rebuild and interactively checks it with the user. +# Confirms if the image should be rebuild and interactively checks it with the user. # In case iit needs to be rebuild. It only ask the user if it determines that the rebuild # is needed and that the rebuild is not already forced. It asks the user using available terminals # So that the script works also from within pre-commit run via git hooks - where stdin is not @@ -1072,7 +1052,7 @@ function build_images::build_prod_images_from_locally_built_airflow_packages() { mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/" # Build apache airflow packages - build_airflow_packages::build_airflow_packages + runs::run_prepare_airflow_packages mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/" build_images::build_prod_images diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 3653b2ea76282..3c5a4d3d8ad6b 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -130,9 +130,6 @@ function initialization::initialize_base_variables() { # Cleans up docker context files if specified export CLEANUP_DOCKER_CONTEXT_FILES="false" - # If set to true, RBAC UI will not be used for 1.10 version - export DISABLE_RBAC=${DISABLE_RBAC:="false"} - # if set to true, the ci image will look for packages in dist folder and will install them # during entering the container export USE_PACKAGES_FROM_DIST=${USE_PACKAGES_FROM_DIST:="false"} @@ -498,6 +495,9 @@ function initialization::initialize_kubernetes_variables() { # Currently supported versions of Helm CURRENT_HELM_VERSIONS+=("v3.2.4") export CURRENT_HELM_VERSIONS + # Current executor in chart + CURRENT_EXECUTOR+=("KubernetesExecutor") + export CURRENT_EXECUTOR # Default Kubernetes version export DEFAULT_KUBERNETES_VERSION="${CURRENT_KUBERNETES_VERSIONS[0]}" # Default Kubernetes mode @@ -506,6 +506,8 @@ function initialization::initialize_kubernetes_variables() { export DEFAULT_KIND_VERSION="${CURRENT_KIND_VERSIONS[0]}" # Default Helm version export DEFAULT_HELM_VERSION="${CURRENT_HELM_VERSIONS[0]}" + # Default airflow executor used in cluster + export DEFAULT_EXECUTOR="${CURRENT_EXECUTOR[0]}" # Namespace where airflow is installed via helm export HELM_AIRFLOW_NAMESPACE="airflow" # Kubernetes version @@ -516,6 +518,8 @@ function initialization::initialize_kubernetes_variables() { export KIND_VERSION=${KIND_VERSION:=${DEFAULT_KIND_VERSION}} # Helm version export HELM_VERSION=${HELM_VERSION:=${DEFAULT_HELM_VERSION}} + # Airflow Executor + export EXECUTOR=${EXECUTOR:=${DEFAULT_EXECUTOR}} # Kubectl version export KUBECTL_VERSION=${KUBERNETES_VERSION:=${DEFAULT_KUBERNETES_VERSION}} # Local Kind path @@ -722,7 +726,6 @@ Initialization variables: LOAD_EXAMPLES: '${LOAD_EXAMPLES}' USE_AIRFLOW_VERSION: '${USE_AIRFLOW_VERSION=}' USE_PACKAGES_FROM_DIST: '${USE_PACKAGES_FROM_DIST=}' - DISABLE_RBAC: '${DISABLE_RBAC}' Test variables: @@ -790,6 +793,7 @@ function initialization::make_constants_read_only() { readonly KUBERNETES_VERSION readonly KIND_VERSION readonly HELM_VERSION + readonly EXECUTOR readonly KUBECTL_VERSION readonly POSTGRES_VERSION @@ -899,6 +903,7 @@ function initialization::make_constants_read_only() { readonly CURRENT_MYSQL_VERSIONS readonly CURRENT_KIND_VERSIONS readonly CURRENT_HELM_VERSIONS + readonly CURRENT_EXECUTOR readonly ALL_PYTHON_MAJOR_MINOR_VERSIONS } diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index a7e75d70d3012..cc42bf7b424bf 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -131,6 +131,10 @@ function kind::perform_kind_cluster_operation() { echo "Kubernetes mode: ${KUBERNETES_MODE}" echo + echo + echo "Executor: ${EXECUTOR}" + echo + if [[ ${OPERATION} == "status" ]]; then if [[ ${ALL_CLUSTERS} == *"${KIND_CLUSTER_NAME}"* ]]; then echo @@ -331,10 +335,9 @@ function kind::deploy_airflow_with_helm() { --set "images.airflow.repository=${DOCKERHUB_USER}/${DOCKERHUB_REPO}" \ --set "images.airflow.tag=${AIRFLOW_PROD_BASE_TAG}-kubernetes" -v 1 \ --set "defaultAirflowTag=${AIRFLOW_PROD_BASE_TAG}-kubernetes" -v 1 \ - --set "config.api.auth_backend=airflow.api.auth.backend.default" \ - --set "config.api.enable_experimental_api=true" \ + --set "config.api.auth_backend=airflow.api.auth.backend.basic_auth" \ --set "config.logging.logging_level=DEBUG" \ - --set "executor=KubernetesExecutor" + --set "executor=${EXECUTOR}" echo popd > /dev/null 2>&1|| exit 1 } diff --git a/scripts/ci/libraries/_md5sum.sh b/scripts/ci/libraries/_md5sum.sh index 052e98d51bd6e..4d4a1e7326cc4 100644 --- a/scripts/ci/libraries/_md5sum.sh +++ b/scripts/ci/libraries/_md5sum.sh @@ -17,7 +17,7 @@ # under the License. # -# Verifies if stored md5sum of the file changed since the last tme ot was checked +# Verifies if stored md5sum of the file changed since the last time it was checked # The md5sum files are stored in .build directory - you can delete this directory # If you want to rebuild everything from the scratch # @@ -55,7 +55,7 @@ function md5sum::calculate_file_md5sum { # # Moves md5sum file from it's temporary location in CACHE_TMP_FILE_DIR to -# BUILD_CACHE_DIR - thus updating stored MD5 sum fo the file +# BUILD_CACHE_DIR - thus updating stored MD5 sum for the file # function md5sum::move_file_md5sum { local FILE="${1}" diff --git a/scripts/ci/libraries/_parameters.sh b/scripts/ci/libraries/_parameters.sh index 6b43d5687689b..c1ee8ee21a133 100644 --- a/scripts/ci/libraries/_parameters.sh +++ b/scripts/ci/libraries/_parameters.sh @@ -31,7 +31,7 @@ function parameters::save_to_file() { # parameters: # $1 - name of the variable # $2 - descriptive name of the parameter -# $3 - flag used to set te parameter +# $3 - flag used to set the parameter function parameters::check_allowed_param() { _VARIABLE_NAME="${1}" _VARIABLE_DESCRIPTIVE_NAME="${2}" diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh index 5b16240428810..6dea2f70aa097 100644 --- a/scripts/ci/libraries/_push_pull_remove_images.sh +++ b/scripts/ci/libraries/_push_pull_remove_images.sh @@ -292,7 +292,7 @@ function push_pull_remove_images::push_prod_images() { } # waits for an image to be available in GitHub Packages. Should be run with `set +e` -# the buid automatically determines which registry to use based one the images available +# the build automatically determines which registry to use based one the images available function push_pull_remove_images::check_for_image_in_github_packages() { local github_repository_lowercase github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')" diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh index 9c7c847f43d8f..2b8d4073397dd 100644 --- a/scripts/ci/libraries/_runs.sh +++ b/scripts/ci/libraries/_runs.sh @@ -45,7 +45,7 @@ function runs::run_prepare_airflow_packages() { -t \ -v "${AIRFLOW_SOURCES}:/opt/airflow" \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/in_container/run_prepare_airflow_packages.sh" "${@}" + "--" "/opt/airflow/scripts/in_container/run_prepare_airflow_packages.sh" start_end::group_end } diff --git a/scripts/ci/libraries/_script_init.sh b/scripts/ci/libraries/_script_init.sh index 12bb3a33b108c..0f3c862a44154 100755 --- a/scripts/ci/libraries/_script_init.sh +++ b/scripts/ci/libraries/_script_init.sh @@ -16,7 +16,14 @@ # specific language governing permissions and limitations # under the License. -set -euo pipefail +set -eo pipefail + +if [[ $(uname -s) != "Darwin" ]]; then + # do not fail with undefined variable on MacOS. The old Bash which is default on Mac OS + # fails with undefined variable when you are passing an empty variable and this causes + # problems for example when you try to pass empty list of arguments "${@}" + set -u +fi export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}" readonly AIRFLOW_SOURCES diff --git a/scripts/ci/libraries/_testing.sh b/scripts/ci/libraries/_testing.sh index 1a050937f564f..11220a8727ce2 100644 --- a/scripts/ci/libraries/_testing.sh +++ b/scripts/ci/libraries/_testing.sh @@ -54,7 +54,7 @@ function testing::get_maximum_parallel_test_jobs() { docker_engine_resources::get_available_cpus_in_docker if [[ -n ${RUNS_ON=} && ${RUNS_ON} != *"self-hosted"* ]]; then echo - echo "${COLOR_YELLOW}This is a Github Public runner - for now we are forcing max parallel Quarantined tests jobs to 1 for those${COLOR_RESET}" + echo "${COLOR_YELLOW}This is a GitHub Public runner - for now we are forcing max parallel Quarantined tests jobs to 1 for those${COLOR_RESET}" echo export MAX_PARALLEL_QUARANTINED_TEST_JOBS="1" else diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index 8f9c74d6f9add..7e77eddec8c7c 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -65,7 +65,7 @@ function output_all_basic_variables() { else initialization::ga_output python-versions \ "$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")" - # this will work as long as DEFAULT_PYTHON_MAJOR_VERSION is the same on HEAD and v1-10 + # this will work as long as DEFAULT_PYTHON_MAJOR_VERSION is the same on HEAD # all-python-versions are used in BuildImage Workflow initialization::ga_output all-python-versions \ "$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")" @@ -345,7 +345,7 @@ function check_if_setup_files_changed() { function check_if_javascript_security_scans_should_be_run() { - start_end::group_start "Check Javascript security scans" + start_end::group_start "Check JavaScript security scans" local pattern_array=( "^airflow/.*\.[jt]sx?" "^airflow/.*\.lock" diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index 28307e3333050..3f1586369f5a8 100755 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -36,18 +36,7 @@ function common::get_airflow_version_specification() { function common::get_constraints_location() { # auto-detect Airflow-constraint reference and location if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE}" ]]; then - if [[ ${AIRFLOW_VERSION} =~ [^0-9]*1[^0-9]*10[^0-9]([0-9]*) ]]; then - # All types of references/versions match this regexp for 1.10 series - # for example v1_10_test, 1.10.10, 1.10.9 etc. ${BASH_REMATCH[1]} matches last - # minor digit of version and it's length is 0 for v1_10_test, 1 for 1.10.9 and 2 for 1.10.10+ - AIRFLOW_MINOR_VERSION_NUMBER=${BASH_REMATCH[1]} - if [[ ${#AIRFLOW_MINOR_VERSION_NUMBER} == "0" ]]; then - # For v1_10_* branches use constraints-1-10 branch - AIRFLOW_CONSTRAINTS_REFERENCE=constraints-1-10 - else - AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION} - fi - elif [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then + if [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION} else AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH} diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index d1982cf771bc0..611edb47695ba 100755 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -29,13 +29,30 @@ function install_airflow_and_providers_from_docker_context_files(){ if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} fi + + # shellcheck disable=SC2206 + local pip_flags=( + # Don't quote this -- if it is empty we don't want it to create an + # empty array element + ${AIRFLOW_INSTALL_USER_FLAG} + --find-links="file:///docker-context-files" + ) + # Find Apache Airflow packages in docker-context files local reinstalling_apache_airflow_package reinstalling_apache_airflow_package=$(ls \ /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true) # Add extras when installing airflow if [[ -n "${reinstalling_apache_airflow_package}" ]]; then - reinstalling_apache_airflow_package="${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]" + # When a provider depends on a dev version of Airflow, we need to + # specify `apache-airflow==$VER`, otherwise pip will look for it on + # pip, and fail to find it + + # This will work as long as the wheel file is correctly named, which it + # will be if it was build by wheel tooling + local ver + ver=$(basename "$reinstalling_apache_airflow_package" | cut -d "-" -f 2) + reinstalling_apache_airflow_package="apache-airflow[${AIRFLOW_EXTRAS}]==$ver" fi # Find Apache Airflow packages in docker-context files @@ -52,12 +69,9 @@ function install_airflow_and_providers_from_docker_context_files(){ echo Force re-installing airflow and providers from local files with eager upgrade echo # force reinstall all airflow + provider package local files with eager upgrade - pip install ${AIRFLOW_INSTALL_USER_FLAG} --force-reinstall --upgrade --upgrade-strategy eager \ + pip install "${pip_flags[@]}" --upgrade --upgrade-strategy eager \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} - # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} else echo echo Force re-installing airflow and providers from local files with constraints and upgrade if needed @@ -69,7 +83,7 @@ function install_airflow_and_providers_from_docker_context_files(){ curl -L "${AIRFLOW_CONSTRAINTS_LOCATION}" | grep -ve '^apache-airflow' > /tmp/constraints.txt fi # force reinstall airflow + provider package local files with constraints + upgrade if needed - pip install ${AIRFLOW_INSTALL_USER_FLAG} --force-reinstall \ + pip install "${pip_flags[@]}" --force-reinstall \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ --constraint /tmp/constraints.txt rm /tmp/constraints.txt @@ -78,11 +92,12 @@ function install_airflow_and_providers_from_docker_context_files(){ # then upgrade if needed without using constraints to account for new limits in setup.py pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} - # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} fi + # make sure correct PIP version is left installed + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + } # Simply install all other (non-apache-airflow) packages placed in docker-context files diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 8bdcc295302d4..d755fe02a53e2 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -561,6 +561,27 @@ function check_missing_providers() { rm "$LIST_OF_DIRS_FILE" } +function rename_packages_if_needed() { + cd "${AIRFLOW_SOURCES}" || exit 1 + pushd dist >/dev/null 2>&1 || exit 1 + if [[ -n "${FILE_VERSION_SUFFIX}" ]]; then + # In case we have FILE_VERSION_SUFFIX we rename prepared files + if [[ "${PACKAGE_FORMAT}" == "sdist" || "${PACKAGE_FORMAT}" == "both" ]]; then + for FILE in *.tar.gz + do + mv "${FILE}" "${FILE//\.tar\.gz/${FILE_VERSION_SUFFIX}.tar.gz}" + done + fi + if [[ "${PACKAGE_FORMAT}" == "wheel" || "${PACKAGE_FORMAT}" == "both" ]]; then + for FILE in *.whl + do + mv "${FILE}" "${FILE//\-py3/${FILE_VERSION_SUFFIX}-py3}" + done + fi + fi + popd >/dev/null || exit 1 +} + function get_providers_to_act_on() { group_start "Get all providers" if [[ -z "$*" ]]; then diff --git a/scripts/in_container/bin/run_tmux b/scripts/in_container/bin/run_tmux index e93988af3f8f4..3c65f3e8760b1 100755 --- a/scripts/in_container/bin/run_tmux +++ b/scripts/in_container/bin/run_tmux @@ -54,7 +54,7 @@ tmux send-keys 'airflow webserver' C-m if [[ -z "${USE_AIRFLOW_VERSION=}" ]]; then tmux select-pane -t 0 tmux split-window -h - tmux send-keys 'cd /opt/airflow/airflow/www/; yarn dev' C-m + tmux send-keys 'cd /opt/airflow/airflow/www/; yarn install --frozen-lockfile; yarn dev' C-m fi # Attach Session, on the Main window diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh index a7b5788e7226a..6ba7f6e1f4de0 100755 --- a/scripts/in_container/entrypoint_ci.sh +++ b/scripts/in_container/entrypoint_ci.sh @@ -19,15 +19,6 @@ if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then set -x fi -function disable_rbac_if_requested() { - if [[ ${DISABLE_RBAC:="false"} == "true" ]]; then - export AIRFLOW__WEBSERVER__RBAC="False" - else - export AIRFLOW__WEBSERVER__RBAC="True" - fi -} - - # shellcheck source=scripts/in_container/_in_container_script_init.sh . /opt/airflow/scripts/in_container/_in_container_script_init.sh @@ -153,8 +144,6 @@ cp -f "${IN_CONTAINER_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg" # Change the default worker_concurrency for tests export AIRFLOW__CELERY__WORKER_CONCURRENCY=8 -disable_rbac_if_requested - set +e "${IN_CONTAINER_DIR}/check_environment.sh" ENVIRONMENT_EXIT_CODE=$? @@ -242,9 +231,14 @@ EXTRA_PYTEST_ARGS=( "-rfEX" ) -if [[ "${TEST_TYPE}" != "Helm" ]]; then +if [[ "${TEST_TYPE}" == "Helm" ]]; then + # Enable parallelism + EXTRA_PYTEST_ARGS+=( + "-n" "auto" + ) +else EXTRA_PYTEST_ARGS+=( - "--with-db-init" + "--with-db-init" ) fi diff --git a/scripts/in_container/run_install_and_test_provider_packages.sh b/scripts/in_container/run_install_and_test_provider_packages.sh index 6dd19084f6843..9dd85407d3fc0 100755 --- a/scripts/in_container/run_install_and_test_provider_packages.sh +++ b/scripts/in_container/run_install_and_test_provider_packages.sh @@ -137,7 +137,7 @@ function discover_all_extra_links() { group_start "Listing available extra links via 'airflow providers links'" COLUMNS=180 airflow providers links - local expected_number_of_extra_links=4 + local expected_number_of_extra_links=6 local actual_number_of_extra_links actual_number_of_extra_links=$(airflow providers links --output table | grep -c ^airflow.providers | xargs) if [[ ${actual_number_of_extra_links} != "${expected_number_of_extra_links}" ]]; then diff --git a/scripts/in_container/run_prepare_airflow_packages.sh b/scripts/in_container/run_prepare_airflow_packages.sh new file mode 100755 index 0000000000000..dcbe283274250 --- /dev/null +++ b/scripts/in_container/run_prepare_airflow_packages.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck source=scripts/in_container/_in_container_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" + +function prepare_airflow_packages() { + echo "-----------------------------------------------------------------------------------" + if [[ "${VERSION_SUFFIX_FOR_PYPI}" == '' && "${VERSION_SUFFIX_FOR_SVN}" == '' + && ${FILE_VERSION_SUFFIX} == '' ]]; then + echo + echo "Preparing official version of provider with no suffixes" + echo + elif [[ ${FILE_VERSION_SUFFIX} != '' ]]; then + echo + echo " Preparing release candidate of providers with file version suffix only (resulting file will be renamed): ${FILE_VERSION_SUFFIX}" + echo + elif [[ "${VERSION_SUFFIX_FOR_PYPI}" == '' ]]; then + echo + echo " Package Version of providers of set for SVN version): ${TARGET_VERSION_SUFFIX}" + echo + elif [[ "${VERSION_SUFFIX_FOR_SVN}" == '' ]]; then + echo + echo " Package Version of providers suffix set for PyPI version: ${TARGET_VERSION_SUFFIX}" + echo + else + # Both SV/PYPI are set to the same version here! + echo + echo " Pre-release version (alpha beta) suffix set in both SVN/PyPI: ${TARGET_VERSION_SUFFIX}" + echo + fi + echo "-----------------------------------------------------------------------------------" + + rm -rf -- *egg-info* + rm -rf -- build + + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" + + local packages=() + + if [[ ${PACKAGE_FORMAT} == "wheel" || ${PACKAGE_FORMAT} == "both" ]] ; then + packages+=("bdist_wheel") + fi + if [[ ${PACKAGE_FORMAT} == "sdist" || ${PACKAGE_FORMAT} == "both" ]] ; then + packages+=("sdist") + fi + local tag_build=() + if [[ -z "${VERSION_SUFFIX_FOR_SVN}" && -n ${VERSION_SUFFIX_FOR_PYPI} || + -n "${VERSION_SUFFIX_FOR_SVN}" && -n "${VERSION_SUFFIX_FOR_PYPI}" ]]; then + # only adds suffix to setup.py if version suffix for PyPI is set but the SVN one is not set + # (so when rc is prepared) + # or when they are both set (so when we prepare alpha/beta/dev) + # + # In case the suffix is already added, we are not adding it again - this is to handle the + # case on where the suffix is embedded during development (we keep NEW_VERSIONdev0 in the code + # for a while during development, but around the release it changes to the final one so our CI + # must handle both cases. + # + # Instead we check if the prefix we want to add is the same as the embedded one and fail if not. + AIRFLOW_VERSION=$(python setup.py --version) + if [[ ${AIRFLOW_VERSION} =~ ^[0-9\.]+$ ]]; then + echo + echo "${COLOR_BLUE}Adding ${VERSION_SUFFIX_FOR_PYPI} suffix to ${AIRFLOW_VERSION}${COLOR_RESET}" + echo + tag_build=('egg_info' '--tag-build' "${VERSION_SUFFIX_FOR_PYPI}") + else + if [[ ${AIRFLOW_VERSION} != *${VERSION_SUFFIX_FOR_PYPI} ]]; then + echo + echo "${COLOR_RED}The requested PyPI suffix ${VERSION_SUFFIX_FOR_PYPI} does not match the one in ${AIRFLOW_VERSION}. Exiting.${COLOR_RESET}" + echo + exit 1 + fi + fi + fi + + # Prepare airflow's wheel + PYTHONUNBUFFERED=1 python setup.py compile_assets "${tag_build[@]}" "${packages[@]}" + + # clean-up + rm -rf -- *egg-info* + rm -rf -- build + + echo + echo "${COLOR_GREEN}Airflow package prepared in format: ${PACKAGE_FORMAT}${COLOR_RESET}" + echo +} + + + +verify_suffix_versions_for_package_preparation + +install_supported_pip_version + +prepare_airflow_packages +rename_packages_if_needed + +echo +echo "${COLOR_GREEN}All good! Airflow packages are prepared in dist folder${COLOR_RESET}" +echo diff --git a/scripts/in_container/run_prepare_provider_packages.sh b/scripts/in_container/run_prepare_provider_packages.sh index 4cb8d507470cb..b24f933de8a33 100755 --- a/scripts/in_container/run_prepare_provider_packages.sh +++ b/scripts/in_container/run_prepare_provider_packages.sh @@ -148,27 +148,6 @@ function build_provider_packages() { fi } -function rename_packages_if_needed() { - cd "${AIRFLOW_SOURCES}" || exit 1 - pushd dist >/dev/null 2>&1 || exit 1 - if [[ -n "${FILE_VERSION_SUFFIX}" ]]; then - # In case we have FILE_VERSION_SUFFIX we rename prepared files - if [[ "${PACKAGE_FORMAT}" == "sdist" || "${PACKAGE_FORMAT}" == "both" ]]; then - for FILE in *.tar.gz - do - mv "${FILE}" "${FILE//\.tar\.gz/${FILE_VERSION_SUFFIX}.tar.gz}" - done - fi - if [[ "${PACKAGE_FORMAT}" == "wheel" || "${PACKAGE_FORMAT}" == "both" ]]; then - for FILE in *.whl - do - mv "${FILE}" "${FILE//\-py3/${FILE_VERSION_SUFFIX}-py3}" - done - fi - fi - popd >/dev/null -} - setup_provider_packages cd "${PROVIDER_PACKAGES_DIR}" || exit 1 diff --git a/setup.cfg b/setup.cfg index bad32b3a8b7c6..72210ea5e21ba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -90,8 +90,9 @@ install_requires = cattrs~=1.1;python_version>"3.6" colorlog>=4.0.2 connexion[swagger-ui,flask]>=2.6.0,<3 - croniter>=0.3.17, <0.4 + croniter>=0.3.17, <1.1 cryptography>=0.9.3 + dataclasses;python_version<"3.7" dill>=0.2.2, <0.4 flask>=1.1.0, <2.0 flask-appbuilder~=3.1,>=3.1.1 diff --git a/setup.py b/setup.py index 9331d1c6b9b0a..7a3024ec6f33c 100644 --- a/setup.py +++ b/setup.py @@ -132,7 +132,7 @@ def git_version(version_: str) -> str: Return a version to identify the state of the underlying git repo. The version will indicate whether the head of the current git-backed working directory is tied to a release tag or not : it will indicate the former with a 'release:{version}' prefix - and the latter with a 'dev0' prefix. Following the prefix will be a sha of the current + and the latter with a '.dev0' suffix. Following the prefix will be a sha of the current branch head. Finally, a "dirty" suffix is appended to indicate that uncommitted changes are present. @@ -263,7 +263,7 @@ def get_sphinx_theme_version() -> str: 'sphinxcontrib-spelling==5.2.1', ] docker = [ - 'docker~=3.0', + 'docker', ] druid = [ 'pydruid>=0.4.1', @@ -484,6 +484,7 @@ def get_sphinx_theme_version() -> str: 'click~=7.1', 'coverage', 'docutils', + 'filelock', 'flake8>=3.6.0', 'flake8-colors', 'flaky', diff --git a/tests/api_connexion/endpoints/test_log_endpoint.py b/tests/api_connexion/endpoints/test_log_endpoint.py index 675ff21aba9e8..6189e430fe58d 100644 --- a/tests/api_connexion/endpoints/test_log_endpoint.py +++ b/tests/api_connexion/endpoints/test_log_endpoint.py @@ -113,6 +113,7 @@ def _prepare_db(self): dagbag = self.app.dag_bag # pylint: disable=no-member dag = DAG(self.DAG_ID, start_date=timezone.parse(self.default_time)) dag.sync_to_db() + dagbag.dags.pop(self.DAG_ID, None) dagbag.bag_dag(dag=dag, root_dag=dag) with create_session() as session: self.ti = TaskInstance( @@ -174,6 +175,7 @@ def test_get_logs_of_removed_task(self, session): # Recreate DAG without tasks dagbag = self.app.dag_bag # pylint: disable=no-member dag = DAG(self.DAG_ID, start_date=timezone.parse(self.default_time)) + del dagbag.dags[self.DAG_ID] dagbag.bag_dag(dag=dag, root_dag=dag) key = self.app.config["SECRET_KEY"] diff --git a/tests/api_connexion/endpoints/test_provider_endpoint.py b/tests/api_connexion/endpoints/test_provider_endpoint.py new file mode 100644 index 0000000000000..693331e7738f5 --- /dev/null +++ b/tests/api_connexion/endpoints/test_provider_endpoint.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from collections import OrderedDict +from unittest import mock + +import pytest + +from airflow.security import permissions +from tests.test_utils.api_connexion_utils import create_user, delete_user + +MOCK_PROVIDERS = OrderedDict( + [ + ( + 'apache-airflow-providers-amazon', + ( + '1.0.0', + { + 'package-name': 'apache-airflow-providers-amazon', + 'name': 'Amazon', + 'description': '`Amazon Web Services (AWS) `__.\n', + 'versions': ['1.0.0'], + }, + ), + ), + ( + 'apache-airflow-providers-apache-cassandra', + ( + '1.0.0', + { + 'package-name': 'apache-airflow-providers-apache-cassandra', + 'name': 'Apache Cassandra', + 'description': '`Apache Cassandra `__.\n', + 'versions': ['1.0.0'], + }, + ), + ), + ] +) + + +@pytest.fixture(scope="module") +def configured_app(minimal_app_for_api): + app = minimal_app_for_api + create_user( + app, # type: ignore + username="test", + role_name="Test", + permissions=[(permissions.ACTION_CAN_READ, permissions.RESOURCE_PROVIDER)], + ) + create_user(app, username="test_no_permissions", role_name="TestNoPermissions") # type: ignore + + yield app + + delete_user(app, username="test") # type: ignore + delete_user(app, username="test_no_permissions") # type: ignore + + +class TestBaseProviderEndpoint: + @pytest.fixture(autouse=True) + def setup_attrs(self, configured_app) -> None: + self.app = configured_app + self.client = self.app.test_client() # type:ignore + + +class TestGetProviders(TestBaseProviderEndpoint): + @mock.patch( + "airflow.providers_manager.ProvidersManager.providers", + new_callable=mock.PropertyMock, + return_value={}, + ) + def test_response_200_empty_list(self, mock_providers): + response = self.client.get("/api/v1/providers", environ_overrides={'REMOTE_USER': "test"}) + assert response.status_code == 200 + assert response.json == {"providers": [], "total_entries": 0} + + @mock.patch( + "airflow.providers_manager.ProvidersManager.providers", + new_callable=mock.PropertyMock, + return_value=MOCK_PROVIDERS, + ) + def test_response_200(self, mock_providers): + response = self.client.get("/api/v1/providers", environ_overrides={'REMOTE_USER': "test"}) + assert response.status_code == 200 + assert response.json == { + 'providers': [ + { + 'description': 'Amazon Web Services (AWS) https://aws.amazon.com/', + 'package_name': 'apache-airflow-providers-amazon', + 'version': '1.0.0', + }, + { + 'description': 'Apache Cassandra http://cassandra.apache.org/', + 'package_name': 'apache-airflow-providers-apache-cassandra', + 'version': '1.0.0', + }, + ], + 'total_entries': 2, + } + + def test_should_raises_401_unauthenticated(self): + response = self.client.get("/api/v1/providers") + assert response.status_code == 401 + + def test_should_raise_403_forbidden(self): + response = self.client.get( + "/api/v1/providers", environ_overrides={'REMOTE_USER': "test_no_permissions"} + ) + assert response.status_code == 403 diff --git a/tests/bats/breeze/test_breeze_complete.bats b/tests/bats/breeze/test_breeze_complete.bats index 3ca32ce4fe78b..c1dfed1659c43 100644 --- a/tests/bats/breeze/test_breeze_complete.bats +++ b/tests/bats/breeze/test_breeze_complete.bats @@ -25,7 +25,7 @@ source "${AIRFLOW_SOURCES}/breeze-complete" breeze_complete::get_known_values_breeze "-p" - assert_equal "${_breeze_known_values}" "2.7 3.5 3.6 3.7 3.8" + assert_equal "${_breeze_known_values}" "3.6 3.7 3.8" } @test "Test get_known_values long" { @@ -34,7 +34,7 @@ source "${AIRFLOW_SOURCES}/breeze-complete" breeze_complete::get_known_values_breeze "--python" - assert_equal "${_breeze_known_values}" "2.7 3.5 3.6 3.7 3.8" + assert_equal "${_breeze_known_values}" "3.6 3.7 3.8" } @test "Test wrong get_known_values" { @@ -125,7 +125,7 @@ COMP_WORDS=("--python" "") breeze_complete::_comp_breeze - assert_equal "${COMPREPLY[*]}" "2.7 3.5 3.6 3.7 3.8" + assert_equal "${COMPREPLY[*]}" "3.6 3.7 3.8" } @test "Test autocomplete --python with prefix" { @@ -136,7 +136,7 @@ COMP_WORDS=("--python" "3") breeze_complete::_comp_breeze - assert_equal "${COMPREPLY[*]}" "3.5 3.6 3.7 3.8" + assert_equal "${COMPREPLY[*]}" "3.6 3.7 3.8" } @test "Test autocomplete build-" { @@ -150,12 +150,12 @@ assert_equal "${COMPREPLY[*]}" "build-docs build-image" } -@test "Test allowed python versions are same as ALL" { +@test "Test allowed python versions are same as CURRENT" { load ../bats_utils #shellcheck source=breeze-complete source "${AIRFLOW_SOURCES}/breeze-complete" - assert_equal "${_breeze_allowed_python_major_minor_versions}" "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[*]}" + assert_equal "${_breeze_allowed_python_major_minor_versions}" "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[*]}" } @test "Test allowed Kubernetes versions same as CURRENT" { diff --git a/tests/core/test_config_templates.py b/tests/core/test_config_templates.py index 2efa838741abd..aa98fa93bf4ef 100644 --- a/tests/core/test_config_templates.py +++ b/tests/core/test_config_templates.py @@ -47,7 +47,6 @@ 'scheduler', 'kerberos', 'github_enterprise', - 'admin', 'elasticsearch', 'elasticsearch_configs', 'kubernetes', @@ -66,7 +65,6 @@ 'smtp', 'celery', 'scheduler', - 'admin', 'elasticsearch', 'elasticsearch_configs', 'kubernetes', diff --git a/tests/core/test_core_to_contrib.py b/tests/core/test_core_to_contrib.py index 4286337b5cbbf..2973702a86ad4 100644 --- a/tests/core/test_core_to_contrib.py +++ b/tests/core/test_core_to_contrib.py @@ -18,6 +18,7 @@ import importlib import sys +import warnings from inspect import isabstract from unittest import TestCase, mock @@ -54,7 +55,8 @@ def skip_test_with_mssql_in_py38(self, path_a="", path_b=""): @staticmethod def get_class_from_path(path_to_class, parent=False): """ - :param parent indicates if "path_to_class" arg is super class + :param path_to_class: the path to the class + :param parent: indicates if "path_to_class" arg is super class """ path, _, class_name = path_to_class.rpartition(".") @@ -74,22 +76,26 @@ def get_class_from_path(path_to_class, parent=False): def test_is_class_deprecated(self, new_module, old_module): self.skip_test_with_mssql_in_py38(new_module, old_module) deprecation_warning_msg = "This class is deprecated." - old_module_class = self.get_class_from_path(old_module) with pytest.warns(DeprecationWarning, match=deprecation_warning_msg) as warnings: + old_module_class = self.get_class_from_path(old_module) + warnings.clear() with mock.patch(f"{new_module}.__init__") as init_mock: init_mock.return_value = None klass = old_module_class() if isinstance(klass, BaseOperator): # In case of operators we are validating that proper stacklevel - # is used (=3 or =4 if @apply_defaults) + # is used (=3) assert len(warnings) >= 1 - assert any(warning.filename == __file__ for warning in warnings) + # For nicer error reporting from pytest, create a static + # list of filenames + files = [warning.filename for warning in warnings] + assert __file__ in files, old_module init_mock.assert_called_once() @parameterized.expand(ALL) def test_is_subclass(self, parent_class_path, sub_class_path): self.skip_test_with_mssql_in_py38(parent_class_path, sub_class_path) - with mock.patch(f"{parent_class_path}.__init__"): + with mock.patch(f"{parent_class_path}.__init__"), warnings.catch_warnings(record=True): parent_class_path = self.get_class_from_path(parent_class_path, parent=True) sub_class_path = self.get_class_from_path(sub_class_path) self.assert_is_subclass(sub_class_path, parent_class_path) diff --git a/tests/core/test_impersonation_tests.py b/tests/core/test_impersonation_tests.py index bc1fa9b04e5c3..f18fa56482dee 100644 --- a/tests/core/test_impersonation_tests.py +++ b/tests/core/test_impersonation_tests.py @@ -114,17 +114,19 @@ def create_user(): @pytest.mark.quarantined class TestImpersonation(unittest.TestCase): - def setUp(self): - check_original_docker_image() - grant_permissions() - add_default_pool_if_not_exists() - self.dagbag = models.DagBag( + @classmethod + def setUpClass(cls): + cls.dagbag = models.DagBag( dag_folder=TEST_DAG_FOLDER, include_examples=False, ) logger.info('Loaded DAGS:') - logger.info(self.dagbag.dagbag_report()) + logger.info(cls.dagbag.dagbag_report()) + def setUp(self): + check_original_docker_image() + grant_permissions() + add_default_pool_if_not_exists() create_user() def tearDown(self): diff --git a/tests/core/test_logging_config.py b/tests/core/test_logging_config.py index c514b0832d18d..5e0c7de01741d 100644 --- a/tests/core/test_logging_config.py +++ b/tests/core/test_logging_config.py @@ -131,6 +131,8 @@ def settings_context(content, directory=None, name='LOGGING_CONFIG'): :param content: The content of the settings file + :param directory: the directory + :param name: str """ initial_logging_config = os.environ.get("AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS", "") try: diff --git a/tests/core/test_providers_manager.py b/tests/core/test_providers_manager.py index 57b581b366c7c..e6f26ce5e16c8 100644 --- a/tests/core/test_providers_manager.py +++ b/tests/core/test_providers_manager.py @@ -227,6 +227,8 @@ EXTRA_LINKS = [ 'airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink', 'airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleLink', + 'airflow.providers.google.cloud.operators.dataproc.DataprocClusterLink', + 'airflow.providers.google.cloud.operators.dataproc.DataprocJobLink', 'airflow.providers.google.cloud.operators.mlengine.AIPlatformConsoleLink', 'airflow.providers.qubole.operators.qubole.QDSLink', ] diff --git a/tests/core/test_sqlalchemy_config.py b/tests/core/test_sqlalchemy_config.py index c4c909bd203f1..de621413c0dd1 100644 --- a/tests/core/test_sqlalchemy_config.py +++ b/tests/core/test_sqlalchemy_config.py @@ -57,6 +57,7 @@ def test_configure_orm_with_default_values( pool_pre_ping=True, pool_recycle=1800, pool_size=5, + isolation_level='READ COMMITTED', ) @patch('airflow.settings.setup_event_handlers') @@ -75,11 +76,15 @@ def test_sql_alchemy_connect_args( } with conf_vars(config): settings.configure_orm() + engine_args = {} + if settings.SQL_ALCHEMY_CONN.startswith('mysql'): + engine_args['isolation_level'] = 'READ COMMITTED' mock_create_engine.assert_called_once_with( settings.SQL_ALCHEMY_CONN, connect_args=SQL_ALCHEMY_CONNECT_ARGS, poolclass=NullPool, encoding='utf-8', + **engine_args, ) @patch('airflow.settings.setup_event_handlers') diff --git a/tests/dags/test_dag_with_no_tags.py b/tests/dags/test_dag_with_no_tags.py index e1a69295eb315..23590a31b531b 100644 --- a/tests/dags/test_dag_with_no_tags.py +++ b/tests/dags/test_dag_with_no_tags.py @@ -27,5 +27,5 @@ "start_date": DEFAULT_DATE, } -with DAG(dag_id="test_only_dummy_tasks", default_args=default_args, schedule_interval='@once') as dag: +with DAG(dag_id="test_dag_with_no_tags", default_args=default_args, schedule_interval='@once') as dag: task_a = DummyOperator(task_id="test_task_a") diff --git a/tests/executors/test_kubernetes_executor.py b/tests/executors/test_kubernetes_executor.py index e449b96eaac4b..57b5af3dd81e1 100644 --- a/tests/executors/test_kubernetes_executor.py +++ b/tests/executors/test_kubernetes_executor.py @@ -25,14 +25,13 @@ import pytest from kubernetes.client import models as k8s +from kubernetes.client.rest import ApiException from urllib3 import HTTPResponse from airflow.utils import timezone from tests.test_utils.config import conf_vars try: - from kubernetes.client.rest import ApiException - from airflow.executors.kubernetes_executor import ( AirflowKubernetesScheduler, KubernetesExecutor, @@ -120,6 +119,71 @@ def test_execution_date_serialize_deserialize(self): assert datetime_obj == new_datetime_obj + @unittest.skipIf(AirflowKubernetesScheduler is None, 'kubernetes python package is not installed') + @mock.patch('airflow.executors.kubernetes_executor.get_kube_client') + @mock.patch('airflow.executors.kubernetes_executor.client') + @mock.patch('airflow.executors.kubernetes_executor.KubernetesJobWatcher') + def test_delete_pod_successfully( + self, mock_watcher, mock_client, mock_kube_client + ): # pylint: disable=unused-argument + pod_id = "my-pod-1" + namespace = "my-namespace-1" + + mock_delete_namespace = mock.MagicMock() + mock_kube_client.return_value.delete_namespaced_pod = mock_delete_namespace + + kube_executor = KubernetesExecutor() + kube_executor.job_id = "test-job-id" + kube_executor.start() + kube_executor.kube_scheduler.delete_pod(pod_id, namespace) + + mock_delete_namespace.assert_called_with(pod_id, namespace, body=mock_client.V1DeleteOptions()) + + @unittest.skipIf(AirflowKubernetesScheduler is None, 'kubernetes python package is not installed') + @mock.patch('airflow.executors.kubernetes_executor.get_kube_client') + @mock.patch('airflow.executors.kubernetes_executor.client') + @mock.patch('airflow.executors.kubernetes_executor.KubernetesJobWatcher') + def test_delete_pod_raises_404( + self, mock_watcher, mock_client, mock_kube_client + ): # pylint: disable=unused-argument + pod_id = "my-pod-1" + namespace = "my-namespace-2" + + mock_delete_namespace = mock.MagicMock() + mock_kube_client.return_value.delete_namespaced_pod = mock_delete_namespace + + # ApiException is raised because status is not 404 + mock_kube_client.return_value.delete_namespaced_pod.side_effect = ApiException(status=400) + kube_executor = KubernetesExecutor() + kube_executor.job_id = "test-job-id" + kube_executor.start() + + with pytest.raises(ApiException): + kube_executor.kube_scheduler.delete_pod(pod_id, namespace) + mock_delete_namespace.assert_called_with(pod_id, namespace, body=mock_client.V1DeleteOptions()) + + @unittest.skipIf(AirflowKubernetesScheduler is None, 'kubernetes python package is not installed') + @mock.patch('airflow.executors.kubernetes_executor.get_kube_client') + @mock.patch('airflow.executors.kubernetes_executor.client') + @mock.patch('airflow.executors.kubernetes_executor.KubernetesJobWatcher') + def test_delete_pod_404_not_raised( + self, mock_watcher, mock_client, mock_kube_client + ): # pylint: disable=unused-argument + pod_id = "my-pod-1" + namespace = "my-namespace-3" + + mock_delete_namespace = mock.MagicMock() + mock_kube_client.return_value.delete_namespaced_pod = mock_delete_namespace + + # ApiException not raised because the status is 404 + mock_kube_client.return_value.delete_namespaced_pod.side_effect = ApiException(status=404) + kube_executor = KubernetesExecutor() + kube_executor.job_id = "test-job-id" + kube_executor.start() + + kube_executor.kube_scheduler.delete_pod(pod_id, namespace) + mock_delete_namespace.assert_called_with(pod_id, namespace, body=mock_client.V1DeleteOptions()) + class TestKubernetesExecutor(unittest.TestCase): """ @@ -160,7 +224,6 @@ def test_run_next_exception(self, mock_get_kube_client, mock_kubernetes_job_watc ('kubernetes', 'pod_template_file'): path, } with conf_vars(config): - kubernetes_executor = self.kubernetes_executor kubernetes_executor.start() # Execute a task while the Api Throws errors diff --git a/tests/jobs/test_local_task_job.py b/tests/jobs/test_local_task_job.py index bb986b2b092cc..9047f8aa18938 100644 --- a/tests/jobs/test_local_task_job.py +++ b/tests/jobs/test_local_task_job.py @@ -27,6 +27,7 @@ from unittest.mock import patch import pytest +from parameterized import parameterized from airflow import settings from airflow.exceptions import AirflowException, AirflowFailException @@ -92,8 +93,7 @@ def test_localtaskjob_essential_attr(self): check_result_2 = [getattr(job1, attr) is not None for attr in essential_attr] assert all(check_result_2) - @patch('os.getpid') - def test_localtaskjob_heartbeat(self, mock_pid): + def test_localtaskjob_heartbeat(self): session = settings.Session() dag = DAG('test_localtaskjob_heartbeat', start_date=DEFAULT_DATE, default_args={'owner': 'owner1'}) @@ -114,19 +114,23 @@ def test_localtaskjob_heartbeat(self, mock_pid): session.commit() job1 = LocalTaskJob(task_instance=ti, ignore_ti_state=True, executor=SequentialExecutor()) + ti.task = op1 + ti.refresh_from_task(op1) + job1.task_runner = StandardTaskRunner(job1) + job1.task_runner.process = mock.Mock() with pytest.raises(AirflowException): job1.heartbeat_callback() # pylint: disable=no-value-for-parameter - mock_pid.return_value = 1 + job1.task_runner.process.pid = 1 ti.state = State.RUNNING ti.hostname = get_hostname() ti.pid = 1 session.merge(ti) session.commit() - + assert ti.pid != os.getpid() job1.heartbeat_callback(session=None) - mock_pid.return_value = 2 + job1.task_runner.process.pid = 2 with pytest.raises(AirflowException): job1.heartbeat_callback() # pylint: disable=no-value-for-parameter @@ -430,6 +434,7 @@ def dummy_return_code(*args, **kwargs): assert ti.state == State.FAILED # task exits with failure state assert failure_callback_called.value == 1 + @pytest.mark.quarantined def test_mark_success_on_success_callback(self): """ Test that ensures that where a task is marked success in the UI @@ -496,9 +501,15 @@ def task_function(ti): assert task_terminated_externally.value == 1 assert not process.is_alive() - def test_process_kill_call_on_failure_callback(self): + @parameterized.expand( + [ + (signal.SIGTERM,), + (signal.SIGKILL,), + ] + ) + def test_process_kill_calls_on_failure_callback(self, signal_type): """ - Test that ensures that when a task is killed with sigterm + Test that ensures that when a task is killed with sigterm or sigkill on_failure_callback gets executed """ # use shared memory value so we can properly track value change even if @@ -547,13 +558,14 @@ def task_function(ti): process = multiprocessing.Process(target=job1.run) process.start() - for _ in range(0, 10): + for _ in range(0, 20): ti.refresh_from_db() - if ti.state == State.RUNNING: + if ti.state == State.RUNNING and ti.pid is not None: break time.sleep(0.2) assert ti.state == State.RUNNING - os.kill(ti.pid, signal.SIGTERM) + assert ti.pid is not None + os.kill(ti.pid, signal_type) process.join(timeout=10) assert failure_callback_called.value == 1 assert task_terminated_externally.value == 1 @@ -584,5 +596,5 @@ def test_number_of_queries_single_loop(self, mock_get_task_runner, return_codes) mock_get_task_runner.return_value.return_code.side_effects = return_codes job = LocalTaskJob(task_instance=ti, executor=MockExecutor()) - with assert_queries_count(13): + with assert_queries_count(15): job.run() diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 016905d285611..954b395f2bf4b 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -2031,7 +2031,7 @@ def test_cleanup_methods_all_called(self, mock_processor_agent): self.scheduler_job.executor = mock.MagicMock(slots_available=8) self.scheduler_job._run_scheduler_loop = mock.MagicMock(side_effect=Exception("oops")) mock_processor_agent.return_value.end.side_effect = Exception("double oops") - self.scheduler_job.executor.end = mock.MagicMock(side_effect=Exception("tripple oops")) + self.scheduler_job.executor.end = mock.MagicMock(side_effect=Exception("triple oops")) with self.assertRaises(Exception): self.scheduler_job.run() @@ -3921,7 +3921,7 @@ def test_do_schedule_max_active_runs_upstream_failed(self): schedule_interval='@once', max_active_runs=1, ) as dag: - # Cant use DummyOperator as that goes straight to success + # Can't use DummyOperator as that goes straight to success task1 = BashOperator(task_id='dummy1', bash_command='true') session = settings.Session() @@ -4034,7 +4034,7 @@ def test_do_schedule_max_active_runs_task_removed(self): schedule_interval='@once', max_active_runs=1, ) as dag: - # Cant use DummyOperator as that goes straight to success + # Can't use DummyOperator as that goes straight to success task1 = BashOperator(task_id='dummy1', bash_command='true') session = settings.Session() @@ -4081,7 +4081,7 @@ def test_do_schedule_max_active_runs_and_manual_trigger(self): schedule_interval='@once', max_active_runs=1, ) as dag: - # Cant use DummyOperator as that goes straight to success + # Can't use DummyOperator as that goes straight to success task1 = BashOperator(task_id='dummy1', bash_command='true') task2 = BashOperator(task_id='dummy2', bash_command='true') diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py index f942b89186b60..dace668b8f37a 100644 --- a/tests/models/test_baseoperator.py +++ b/tests/models/test_baseoperator.py @@ -27,9 +27,8 @@ from airflow.exceptions import AirflowException from airflow.lineage.entities import File from airflow.models import DAG -from airflow.models.baseoperator import chain, cross_downstream +from airflow.models.baseoperator import BaseOperatorMeta, chain, cross_downstream from airflow.operators.dummy import DummyOperator -from airflow.utils.decorators import apply_defaults from tests.models import DEFAULT_DATE from tests.test_utils.mock_operators import DeprecatedOperator, MockNamedTuple, MockOperator @@ -60,7 +59,56 @@ def __ne__(self, other): setattr(object1, 'ref', object2) +# Essentially similar to airflow.models.baseoperator.BaseOperator +class DummyClass(metaclass=BaseOperatorMeta): + def __init__(self, test_param, params=None, default_args=None): # pylint: disable=unused-argument + self.test_param = test_param + + def set_xcomargs_dependencies(self): + ... + + +class DummySubClass(DummyClass): + def __init__(self, test_sub_param, **kwargs): + super().__init__(**kwargs) + self.test_sub_param = test_sub_param + + class TestBaseOperator(unittest.TestCase): + def test_apply(self): + dummy = DummyClass(test_param=True) + assert dummy.test_param + + with pytest.raises(AirflowException, match='Argument.*test_param.*required'): + DummySubClass(test_sub_param=True) + + def test_default_args(self): + default_args = {'test_param': True} + dummy_class = DummyClass(default_args=default_args) # pylint: disable=no-value-for-parameter + assert dummy_class.test_param + + default_args = {'test_param': True, 'test_sub_param': True} + dummy_subclass = DummySubClass(default_args=default_args) # pylint: disable=no-value-for-parameter + assert dummy_class.test_param + assert dummy_subclass.test_sub_param + + default_args = {'test_param': True} + dummy_subclass = DummySubClass(default_args=default_args, test_sub_param=True) + assert dummy_class.test_param + assert dummy_subclass.test_sub_param + + with pytest.raises(AirflowException, match='Argument.*test_sub_param.*required'): + DummySubClass(default_args=default_args) # pylint: disable=no-value-for-parameter + + def test_incorrect_default_args(self): + default_args = {'test_param': True, 'extra_param': True} + dummy_class = DummyClass(default_args=default_args) # pylint: disable=no-value-for-parameter + assert dummy_class.test_param + + default_args = {'random_params': True} + with pytest.raises(AirflowException, match='Argument.*test_param.*required'): + DummyClass(default_args=default_args) # pylint: disable=no-value-for-parameter + @parameterized.expand( [ ("{{ foo }}", {"foo": "bar"}, "bar"), @@ -135,6 +183,34 @@ def test_render_template(self, content, context, expected_output): result = task.render_template(content, context) assert result == expected_output + @parameterized.expand( + [ + ("{{ foo }}", {"foo": "bar"}, "bar"), + ("{{ foo }}", {"foo": ["bar1", "bar2"]}, ["bar1", "bar2"]), + (["{{ foo }}", "{{ foo | length}}"], {"foo": ["bar1", "bar2"]}, [['bar1', 'bar2'], 2]), + (("{{ foo }}_1", "{{ foo }}_2"), {"foo": "bar"}, ("bar_1", "bar_2")), + ("{{ ds }}", {"ds": date(2018, 12, 6)}, date(2018, 12, 6)), + (datetime(2018, 12, 6, 10, 55), {"foo": "bar"}, datetime(2018, 12, 6, 10, 55)), + ("{{ ds }}", {"ds": datetime(2018, 12, 6, 10, 55)}, datetime(2018, 12, 6, 10, 55)), + (MockNamedTuple("{{ foo }}_1", "{{ foo }}_2"), {"foo": "bar"}, MockNamedTuple("bar_1", "bar_2")), + ( + ("{{ foo }}", "{{ foo.isoformat() }}"), + {"foo": datetime(2018, 12, 6, 10, 55)}, + (datetime(2018, 12, 6, 10, 55), '2018-12-06T10:55:00'), + ), + (None, {}, None), + ([], {}, []), + ({}, {}, {}), + ] + ) + def test_render_template_with_native_envs(self, content, context, expected_output): + """Test render_template given various input types with Native Python types""" + with DAG("test-dag", start_date=DEFAULT_DATE, render_template_as_native_obj=True): + task = DummyOperator(task_id="op1") + + result = task.render_template(content, context) + assert result == expected_output + def test_render_template_fields(self): """Verify if operator attributes are correctly templated.""" with DAG("test-dag", start_date=DEFAULT_DATE): @@ -149,6 +225,20 @@ def test_render_template_fields(self): assert task.arg1 == "footemplated" assert task.arg2 == "bartemplated" + def test_render_template_fields_native_envs(self): + """Verify if operator attributes are correctly templated to Native Python objects.""" + with DAG("test-dag", start_date=DEFAULT_DATE, render_template_as_native_obj=True): + task = MockOperator(task_id="op1", arg1="{{ foo }}", arg2="{{ bar }}") + + # Assert nothing is templated yet + assert task.arg1 == "{{ foo }}" + assert task.arg2 == "{{ bar }}" + + # Trigger templating and verify if attributes are templated correctly + task.render_template_fields(context={"foo": ["item1", "item2"], "bar": 3}) + assert task.arg1 == ["item1", "item2"] + assert task.arg2 == 3 + @parameterized.expand( [ ({"user_defined_macros": {"foo": "bar"}}, "{{ foo }}", {}, "bar"), @@ -228,6 +318,15 @@ def test_jinja_env_creation(self, mock_jinja_env): task.render_template_fields(context={"foo": "whatever", "bar": "whatever"}) assert mock_jinja_env.call_count == 1 + @mock.patch("airflow.models.dag.NativeEnvironment", autospec=True) + def test_jinja_env_creation_native_environment(self, mock_jinja_env): + """Verify if a Jinja environment is created only once when templating.""" + with DAG("test-dag", start_date=DEFAULT_DATE, render_template_as_native_obj=True): + task = MockOperator(task_id="op1", arg1="{{ foo }}", arg2="{{ bar }}") + + task.render_template_fields(context={"foo": "whatever", "bar": "whatever"}) + assert mock_jinja_env.call_count == 1 + def test_set_jinja_env_additional_option(self): """Test render_template given various input types.""" with DAG( @@ -362,7 +461,6 @@ def test_warnings_are_properly_propagated(self): class CustomOp(DummyOperator): template_fields = ("field", "field2") - @apply_defaults def __init__(self, field=None, field2=None, **kwargs): super().__init__(**kwargs) self.field = field diff --git a/tests/models/test_connection.py b/tests/models/test_connection.py index 21fb7aad618b2..29eed1c5be0eb 100644 --- a/tests/models/test_connection.py +++ b/tests/models/test_connection.py @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. import json +import os import re import unittest from collections import namedtuple @@ -61,6 +62,10 @@ def uri_test_name(func, num, param): class TestConnection(unittest.TestCase): def setUp(self): crypto._fernet = None + patcher = mock.patch('airflow.models.connection.mask_secret', autospec=True) + self.mask_secret = patcher.start() + + self.addCleanup(patcher.stop) def tearDown(self): crypto._fernet = None @@ -359,6 +364,15 @@ def test_connection_from_uri(self, test_config: UriTestCaseConfig): else: assert expected_val == actual_val + expected_calls = [] + if test_config.test_conn_attributes.get('password'): + expected_calls.append(mock.call(test_config.test_conn_attributes['password'])) + + if test_config.test_conn_attributes.get('extra_dejson'): + expected_calls.append(mock.call(test_config.test_conn_attributes['extra_dejson'])) + + self.mask_secret.assert_has_calls(expected_calls) + # pylint: disable=undefined-variable @parameterized.expand([(x,) for x in test_from_uri_params], UriTestCaseConfig.uri_test_name) def test_connection_get_uri_from_uri(self, test_config: UriTestCaseConfig): @@ -501,6 +515,8 @@ def test_using_env_var(self): assert 'password' == conn.password assert 5432 == conn.port + self.mask_secret.assert_called_once_with('password') + @mock.patch.dict( 'os.environ', { @@ -601,3 +617,35 @@ def test_connection_mixed(self): ), ): Connection(conn_id="TEST_ID", uri="mysql://", schema="AAA") + + def test_masking_from_db(self): + """Test secrets are masked when loaded directly from the DB""" + from airflow.settings import Session + + session = Session() + + try: + conn = Connection( + conn_id=f"test-{os.getpid()}", + conn_type="http", + password="s3cr3t", + extra='{"apikey":"masked too"}', + ) + session.add(conn) + session.flush() + + # Make sure we re-load it, not just get the cached object back + session.expunge(conn) + + self.mask_secret.reset_mock() + + from_db = session.query(Connection).get(conn.id) + from_db.extra_dejson + + assert self.mask_secret.mock_calls == [ + # We should have called it _again_ when loading from the DB + mock.call("s3cr3t"), + mock.call({"apikey": "masked too"}), + ] + finally: + session.rollback() diff --git a/tests/models/test_dagbag.py b/tests/models/test_dagbag.py index 3c9589640009b..359cd5c11b18f 100644 --- a/tests/models/test_dagbag.py +++ b/tests/models/test_dagbag.py @@ -137,6 +137,38 @@ def test_process_file_that_contains_multi_bytes_char(self): dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False) assert [] == dagbag.process_file(f.name) + def test_process_file_duplicated_dag_id(self): + """Loading a DAG with ID that already existed in a DAG bag should result in an import error.""" + dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False) + + def create_dag(): + from airflow.decorators import dag + + @dag(default_args={'owner': 'owner1'}) + def my_flow(): + pass + + my_dag = my_flow() # noqa # pylint: disable=unused-variable + + source_lines = [line[12:] for line in inspect.getsource(create_dag).splitlines(keepends=True)[1:]] + with NamedTemporaryFile("w+", encoding="utf8") as tf_1, NamedTemporaryFile( + "w+", encoding="utf8" + ) as tf_2: + tf_1.writelines(source_lines) + tf_2.writelines(source_lines) + tf_1.flush() + tf_2.flush() + + found_1 = dagbag.process_file(tf_1.name) + assert len(found_1) == 1 and found_1[0].dag_id == "my_flow" + assert dagbag.import_errors == {} + dags_in_bag = dagbag.dags + + found_2 = dagbag.process_file(tf_2.name) + assert len(found_2) == 0 + assert dagbag.import_errors[tf_2.name].startswith("Ignoring DAG") + assert dagbag.dags == dags_in_bag # Should not change. + def test_zip_skip_log(self): """ test the loading of a DAG from within a zip file that skips another file because diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index f7547535a030b..d83f542f875b9 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -229,8 +229,9 @@ def test_write(self): ) == result_updated @mock.patch.dict(os.environ, {"AIRFLOW_IS_K8S_EXECUTOR_POD": "True"}) + @mock.patch('airflow.utils.log.secrets_masker.redact', autospec=True, side_effect=lambda d, _=None: d) @mock.patch("airflow.settings.pod_mutation_hook") - def test_get_k8s_pod_yaml(self, mock_pod_mutation_hook): + def test_get_k8s_pod_yaml(self, mock_pod_mutation_hook, redact): """ Test that k8s_pod_yaml is rendered correctly, stored in the Database, and are correctly fetched using RTIF.get_k8s_pod_yaml @@ -289,6 +290,8 @@ def test_get_k8s_pod_yaml(self, mock_pod_mutation_hook): } assert expected_pod_yaml == rtif.k8s_pod_yaml + # K8s pod spec dict was passed to redact + redact.assert_any_call(rtif.k8s_pod_yaml) with create_session() as session: session.add(rtif) @@ -303,3 +306,26 @@ def test_get_k8s_pod_yaml(self, mock_pod_mutation_hook): ti2 = TI(task_2, EXECUTION_DATE) assert RTIF.get_k8s_pod_yaml(ti=ti2) is None + + @mock.patch.dict(os.environ, {"AIRFLOW_VAR_API_KEY": "secret"}) + @mock.patch('airflow.utils.log.secrets_masker.redact', autospec=True) + def test_redact(self, redact): + dag = DAG("test_ritf_redact", start_date=START_DATE) + with dag: + task = BashOperator( + task_id="test", + bash_command="echo {{ var.value.api_key }}", + env={'foo': 'secret', 'other_api_key': 'masked based on key name'}, + ) + + redact.side_effect = [ + 'val 1', + 'val 2', + ] + + ti = TI(task=task, execution_date=EXECUTION_DATE) + rtif = RTIF(ti=ti) + assert rtif.rendered_fields == { + 'bash_command': 'val 1', + 'env': 'val 2', + } diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 3184992a585b6..35c00fe8368cf 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -2080,8 +2080,8 @@ def tearDown(self) -> None: @parameterized.expand( [ # Expected queries, mark_success - (10, False), - (5, True), + (12, False), + (7, True), ] ) def test_execute_queries_count(self, expected_query_count, mark_success): @@ -2117,7 +2117,7 @@ def test_execute_queries_count_store_serialized(self): session=session, ) - with assert_queries_count(10): + with assert_queries_count(12): ti._run_raw_task() def test_operator_field_with_serialization(self): diff --git a/tests/models/test_variable.py b/tests/models/test_variable.py index e5e1d7aeda605..28aa0561a5f5f 100644 --- a/tests/models/test_variable.py +++ b/tests/models/test_variable.py @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. +import os import unittest from unittest import mock @@ -33,9 +34,16 @@ class TestVariable(unittest.TestCase): def setUp(self): crypto._fernet = None db.clear_db_variables() + patcher = mock.patch('airflow.models.variable.mask_secret', autospec=True) + self.mask_secret = patcher.start() + + self.addCleanup(patcher.stop) def tearDown(self): crypto._fernet = None + + @classmethod + def tearDownClass(cls): db.clear_db_variables() @conf_vars({('core', 'fernet_key'): ''}) @@ -48,6 +56,9 @@ def test_variable_no_encryption(self): test_var = session.query(Variable).filter(Variable.key == 'key').one() assert not test_var.is_encrypted assert test_var.val == 'value' + # We always call mask_secret for variables, and let the SecretsMasker decide based on the name if it + # should mask anything. That logic is tested in test_secrets_masker.py + self.mask_secret.assert_called_once_with('value', 'key') @conf_vars({('core', 'fernet_key'): Fernet.generate_key().decode()}) def test_variable_with_encryption(self): @@ -172,3 +183,32 @@ def test_variable_delete(self): Variable.delete(key) with pytest.raises(KeyError): Variable.get(key) + + def test_masking_from_db(self): + """Test secrets are masked when loaded directly from the DB""" + + # Normally people will use `Variable.get`, but just in case, catch direct DB access too + + session = settings.Session() + + try: + var = Variable( + key=f"password-{os.getpid()}", + val="s3cr3t", + ) + session.add(var) + session.flush() + + # Make sure we re-load it, not just get the cached object back + session.expunge(var) + + self.mask_secret.reset_mock() + + session.query(Variable).get(var.id) + + assert self.mask_secret.mock_calls == [ + # We should have called it _again_ when loading from the DB + mock.call("s3cr3t", var.key), + ] + finally: + session.rollback() diff --git a/tests/providers/amazon/aws/hooks/test_datasync.py b/tests/providers/amazon/aws/hooks/test_datasync.py index eccdd5fa61ccc..586284492d232 100644 --- a/tests/providers/amazon/aws/hooks/test_datasync.py +++ b/tests/providers/amazon/aws/hooks/test_datasync.py @@ -43,7 +43,7 @@ def test_get_conn(self): # mock_get_conn. We then override it to just return the locally created self.client instead of # the one created by the AWS self.hook. -# Unfortunately this means we cant test the get_conn method - which is why we have it in a +# Unfortunately this means we can't test the get_conn method - which is why we have it in a # separate class above diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_processing.py b/tests/providers/amazon/aws/operators/test_sagemaker_processing.py index 6ceb286b25289..58f0a86b11a2a 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_processing.py +++ b/tests/providers/amazon/aws/operators/test_sagemaker_processing.py @@ -183,8 +183,9 @@ def test_execute_with_existing_job_fail( @mock.patch.object(SageMakerHook, "get_conn") def test_action_if_job_exists_validation(self, mock_client): - sagemaker = SageMakerProcessingOperator( - **self.processing_config_kwargs, config=create_processing_params - ) with pytest.raises(AirflowException): - sagemaker.__init__(action_if_job_exists="not_fail_or_increment") + SageMakerProcessingOperator( + **self.processing_config_kwargs, + config=create_processing_params, + action_if_job_exists="not_fail_or_increment", + ) diff --git a/tests/providers/amazon/aws/transfers/test_mongo_to_s3.py b/tests/providers/amazon/aws/transfers/test_mongo_to_s3.py index 9e9811aa6302e..b0bfb9aab694c 100644 --- a/tests/providers/amazon/aws/transfers/test_mongo_to_s3.py +++ b/tests/providers/amazon/aws/transfers/test_mongo_to_s3.py @@ -94,7 +94,7 @@ def test_execute(self, mock_s3_hook, mock_mongo_hook): operator.execute(None) mock_mongo_hook.return_value.find.assert_called_once_with( - mongo_collection=MONGO_COLLECTION, query=MONGO_QUERY, mongo_db=None, allowDiskUse=False + mongo_collection=MONGO_COLLECTION, query=MONGO_QUERY, mongo_db=None ) op_stringify = self.mock_operator._stringify @@ -117,7 +117,7 @@ def test_execute_compress(self, mock_s3_hook, mock_mongo_hook): operator.execute(None) mock_mongo_hook.return_value.find.assert_called_once_with( - allowDiskUse=False, mongo_collection=MONGO_COLLECTION, query=MONGO_QUERY, mongo_db=None + mongo_collection=MONGO_COLLECTION, query=MONGO_QUERY, mongo_db=None ) op_stringify = self.mock_operator._stringify diff --git a/tests/providers/apache/hive/transfers/test_s3_to_hive.py b/tests/providers/apache/hive/transfers/test_s3_to_hive.py index f22d1863e3d99..4ca6f39a67abc 100644 --- a/tests/providers/apache/hive/transfers/test_s3_to_hive.py +++ b/tests/providers/apache/hive/transfers/test_s3_to_hive.py @@ -162,25 +162,25 @@ def test__get_top_row_as_list(self): self.kwargs['delimiter'] = '\t' fn_txt = self._get_fn('.txt', True) header_list = S3ToHiveOperator(**self.kwargs)._get_top_row_as_list(fn_txt) - assert header_list == ['Sno', 'Some,Text'], "Top row from file doesnt matched expected value" + assert header_list == ['Sno', 'Some,Text'], "Top row from file doesn't matched expected value" self.kwargs['delimiter'] = ',' header_list = S3ToHiveOperator(**self.kwargs)._get_top_row_as_list(fn_txt) - assert header_list == ['Sno\tSome', 'Text'], "Top row from file doesnt matched expected value" + assert header_list == ['Sno\tSome', 'Text'], "Top row from file doesn't matched expected value" def test__match_headers(self): self.kwargs['field_dict'] = OrderedDict([('Sno', 'BIGINT'), ('Some,Text', 'STRING')]) assert S3ToHiveOperator(**self.kwargs)._match_headers( ['Sno', 'Some,Text'] - ), "Header row doesnt match expected value" + ), "Header row doesn't match expected value" # Testing with different column order assert not S3ToHiveOperator(**self.kwargs)._match_headers( ['Some,Text', 'Sno'] - ), "Header row doesnt match expected value" + ), "Header row doesn't match expected value" # Testing with extra column in header assert not S3ToHiveOperator(**self.kwargs)._match_headers( ['Sno', 'Some,Text', 'ExtraColumn'] - ), "Header row doesnt match expected value" + ), "Header row doesn't match expected value" def test__delete_top_row_and_compress(self): s32hive = S3ToHiveOperator(**self.kwargs) diff --git a/tests/providers/google/cloud/hooks/test_bigquery.py b/tests/providers/google/cloud/hooks/test_bigquery.py index f7b09be9cfeb1..79057419b0cf6 100644 --- a/tests/providers/google/cloud/hooks/test_bigquery.py +++ b/tests/providers/google/cloud/hooks/test_bigquery.py @@ -676,6 +676,178 @@ def test_get_schema(self, mock_client): assert "fields" in result assert len(result["fields"]) == 2 + @mock.patch('airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.get_schema') + @mock.patch('airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.update_table') + def test_update_table_schema_with_policy_tags(self, mock_update, mock_get_schema): + mock_get_schema.return_value = { + "fields": [ + {'name': 'emp_name', 'type': 'STRING', 'mode': 'REQUIRED'}, + { + 'name': 'salary', + 'type': 'INTEGER', + 'mode': 'REQUIRED', + 'policyTags': {'names': ['sensitive']}, + }, + {'name': 'not_changed', 'type': 'INTEGER', 'mode': 'REQUIRED'}, + { + 'name': 'subrecord', + 'type': 'RECORD', + 'mode': 'REQUIRED', + 'fields': [ + { + 'name': 'field_1', + 'type': 'STRING', + 'mode': 'REQUIRED', + 'policyTags': {'names': ['sensitive']}, + }, + ], + }, + ] + } + + schema_fields_updates = [ + {'name': 'emp_name', 'description': 'Name of employee', 'policyTags': {'names': ['sensitive']}}, + { + 'name': 'salary', + 'description': 'Monthly salary in USD', + 'policyTags': {}, + }, + { + 'name': 'subrecord', + 'description': 'Some Desc', + 'fields': [ + {'name': 'field_1', 'description': 'Some nested desc'}, + ], + }, + ] + + expected_result_schema = { + 'fields': [ + { + 'name': 'emp_name', + 'type': 'STRING', + 'mode': 'REQUIRED', + 'description': 'Name of employee', + 'policyTags': {'names': ['sensitive']}, + }, + { + 'name': 'salary', + 'type': 'INTEGER', + 'mode': 'REQUIRED', + 'description': 'Monthly salary in USD', + 'policyTags': {}, + }, + {'name': 'not_changed', 'type': 'INTEGER', 'mode': 'REQUIRED'}, + { + 'name': 'subrecord', + 'type': 'RECORD', + 'mode': 'REQUIRED', + 'description': 'Some Desc', + 'fields': [ + { + 'name': 'field_1', + 'type': 'STRING', + 'mode': 'REQUIRED', + 'description': 'Some nested desc', + 'policyTags': {'names': ['sensitive']}, + } + ], + }, + ] + } + + self.hook.update_table_schema( + schema_fields_updates=schema_fields_updates, + include_policy_tags=True, + dataset_id=DATASET_ID, + table_id=TABLE_ID, + ) + + mock_update.assert_called_once_with( + dataset_id=DATASET_ID, + table_id=TABLE_ID, + project_id=PROJECT_ID, + table_resource={'schema': expected_result_schema}, + fields=['schema'], + ) + + @mock.patch('airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.get_schema') + @mock.patch('airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.update_table') + def test_update_table_schema_without_policy_tags(self, mock_update, mock_get_schema): + mock_get_schema.return_value = { + "fields": [ + {'name': 'emp_name', 'type': 'STRING', 'mode': 'REQUIRED'}, + {'name': 'salary', 'type': 'INTEGER', 'mode': 'REQUIRED'}, + {'name': 'not_changed', 'type': 'INTEGER', 'mode': 'REQUIRED'}, + { + 'name': 'subrecord', + 'type': 'RECORD', + 'mode': 'REQUIRED', + 'fields': [ + {'name': 'field_1', 'type': 'STRING', 'mode': 'REQUIRED'}, + ], + }, + ] + } + + schema_fields_updates = [ + {'name': 'emp_name', 'description': 'Name of employee'}, + { + 'name': 'salary', + 'description': 'Monthly salary in USD', + 'policyTags': {'names': ['sensitive']}, + }, + { + 'name': 'subrecord', + 'description': 'Some Desc', + 'fields': [ + {'name': 'field_1', 'description': 'Some nested desc'}, + ], + }, + ] + + expected_result_schema = { + 'fields': [ + {'name': 'emp_name', 'type': 'STRING', 'mode': 'REQUIRED', 'description': 'Name of employee'}, + { + 'name': 'salary', + 'type': 'INTEGER', + 'mode': 'REQUIRED', + 'description': 'Monthly salary in USD', + }, + {'name': 'not_changed', 'type': 'INTEGER', 'mode': 'REQUIRED'}, + { + 'name': 'subrecord', + 'type': 'RECORD', + 'mode': 'REQUIRED', + 'description': 'Some Desc', + 'fields': [ + { + 'name': 'field_1', + 'type': 'STRING', + 'mode': 'REQUIRED', + 'description': 'Some nested desc', + } + ], + }, + ] + } + + self.hook.update_table_schema( + schema_fields_updates=schema_fields_updates, + include_policy_tags=False, + dataset_id=DATASET_ID, + table_id=TABLE_ID, + ) + + mock_update.assert_called_once_with( + dataset_id=DATASET_ID, + table_id=TABLE_ID, + project_id=PROJECT_ID, + table_resource={'schema': expected_result_schema}, + fields=['schema'], + ) + @mock.patch("airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.get_service") def test_invalid_source_format(self, mock_get_service): with pytest.raises( diff --git a/tests/providers/google/cloud/operators/test_automl.py b/tests/providers/google/cloud/operators/test_automl.py index 4c80703d58a1c..bcbacace02dfd 100644 --- a/tests/providers/google/cloud/operators/test_automl.py +++ b/tests/providers/google/cloud/operators/test_automl.py @@ -129,13 +129,14 @@ def test_execute(self, mock_hook): project_id=GCP_PROJECT_ID, payload=PAYLOAD, task_id=TASK_ID, + operation_params={"TEST_KEY": "TEST_VALUE"}, ) op.execute(context=None) mock_hook.return_value.predict.assert_called_once_with( location=GCP_LOCATION, metadata=None, model_id=MODEL_ID, - params={}, + params={"TEST_KEY": "TEST_VALUE"}, payload=PAYLOAD, project_id=GCP_PROJECT_ID, retry=None, diff --git a/tests/providers/google/cloud/operators/test_bigquery.py b/tests/providers/google/cloud/operators/test_bigquery.py index 61030f659ed11..801034c06ce66 100644 --- a/tests/providers/google/cloud/operators/test_bigquery.py +++ b/tests/providers/google/cloud/operators/test_bigquery.py @@ -46,6 +46,7 @@ BigQueryPatchDatasetOperator, BigQueryUpdateDatasetOperator, BigQueryUpdateTableOperator, + BigQueryUpdateTableSchemaOperator, BigQueryUpsertTableOperator, BigQueryValueCheckOperator, ) @@ -290,6 +291,36 @@ def test_execute(self, mock_hook): ) +class TestBigQueryUpdateTableSchemaOperator(unittest.TestCase): + @mock.patch('airflow.providers.google.cloud.operators.bigquery.BigQueryHook') + def test_execute(self, mock_hook): + + schema_field_updates = [ + { + 'name': 'emp_name', + 'description': 'Name of employee', + } + ] + + operator = BigQueryUpdateTableSchemaOperator( + schema_fields_updates=schema_field_updates, + include_policy_tags=False, + task_id=TASK_ID, + dataset_id=TEST_DATASET, + table_id=TEST_TABLE_ID, + project_id=TEST_GCP_PROJECT_ID, + ) + operator.execute(None) + + mock_hook.return_value.update_table_schema.assert_called_once_with( + schema_fields_updates=schema_field_updates, + include_policy_tags=False, + dataset_id=TEST_DATASET, + table_id=TEST_TABLE_ID, + project_id=TEST_GCP_PROJECT_ID, + ) + + class TestBigQueryPatchDatasetOperator(unittest.TestCase): @mock.patch('airflow.providers.google.cloud.operators.bigquery.BigQueryHook') def test_execute(self, mock_hook): diff --git a/tests/providers/google/cloud/operators/test_cloud_sql.py b/tests/providers/google/cloud/operators/test_cloud_sql.py index 3a224788f288f..46a30611734bb 100644 --- a/tests/providers/google/cloud/operators/test_cloud_sql.py +++ b/tests/providers/google/cloud/operators/test_cloud_sql.py @@ -149,9 +149,7 @@ def test_instance_create(self, mock_hook, _check_if_instance_exists): op = CloudSQLCreateInstanceOperator( project_id=PROJECT_ID, instance=INSTANCE_NAME, body=CREATE_BODY, task_id="id" ) - result = op.execute( - context={'task_instance': mock.Mock()} # pylint: disable=assignment-from-no-return - ) + op.execute(context={'task_instance': mock.Mock()}) mock_hook.assert_called_once_with( api_version="v1beta4", gcp_conn_id="google_cloud_default", @@ -160,7 +158,6 @@ def test_instance_create(self, mock_hook, _check_if_instance_exists): mock_hook.return_value.create_instance.assert_called_once_with( project_id=PROJECT_ID, body=CREATE_BODY ) - assert result is None @mock.patch( "airflow.providers.google.cloud.operators.cloud_sql" @@ -171,16 +168,13 @@ def test_instance_create_missing_project_id(self, mock_hook, _check_if_instance_ _check_if_instance_exists.return_value = False mock_hook.return_value.create_instance.return_value = True op = CloudSQLCreateInstanceOperator(instance=INSTANCE_NAME, body=CREATE_BODY, task_id="id") - result = op.execute( - context={'task_instance': mock.Mock()} # pylint: disable=assignment-from-no-return - ) + op.execute(context={'task_instance': mock.Mock()}) mock_hook.assert_called_once_with( api_version="v1beta4", gcp_conn_id="google_cloud_default", impersonation_chain=None, ) mock_hook.return_value.create_instance.assert_called_once_with(project_id=None, body=CREATE_BODY) - assert result is None @mock.patch( "airflow.providers.google.cloud.operators.cloud_sql" @@ -193,16 +187,13 @@ def test_instance_create_idempotent(self, mock_hook, _check_if_instance_exists): op = CloudSQLCreateInstanceOperator( project_id=PROJECT_ID, instance=INSTANCE_NAME, body=CREATE_BODY, task_id="id" ) - result = op.execute( - context={'task_instance': mock.Mock()} # pylint: disable=assignment-from-no-return - ) + op.execute(context={'task_instance': mock.Mock()}) mock_hook.assert_called_once_with( api_version="v1beta4", gcp_conn_id="google_cloud_default", impersonation_chain=None, ) mock_hook.return_value.create_instance.assert_not_called() - assert result is None @mock.patch("airflow.providers.google.cloud.operators.cloud_sql.CloudSQLHook") def test_create_should_throw_ex_when_empty_project_id(self, mock_hook): diff --git a/tests/providers/google/cloud/operators/test_dataflow.py b/tests/providers/google/cloud/operators/test_dataflow.py index 5e7079fb92069..1d5028c06634f 100644 --- a/tests/providers/google/cloud/operators/test_dataflow.py +++ b/tests/providers/google/cloud/operators/test_dataflow.py @@ -21,6 +21,7 @@ from copy import deepcopy from unittest import mock +import airflow from airflow.providers.google.cloud.operators.dataflow import ( CheckJobRunning, DataflowCreateJavaJobOperator, @@ -103,6 +104,7 @@ def setUp(self): poll_sleep=POLL_SLEEP, location=TEST_LOCATION, ) + self.expected_airflow_version = 'v' + airflow.version.version.replace(".", "-").replace("+", "-") def test_init(self): """Test DataFlowPythonOperator instance is properly initialized.""" @@ -149,7 +151,7 @@ def test_exec(self, gcs_hook, dataflow_hook_mock, beam_hook_mock, mock_callback_ "job_name": job_name, "region": TEST_LOCATION, 'output': 'gs://test/output', - 'labels': {'foo': 'bar', 'airflow-version': 'v2-1-0-dev0'}, + 'labels': {'foo': 'bar', 'airflow-version': self.expected_airflow_version}, } start_python_mock.assert_called_once_with( variables=expected_options, @@ -181,6 +183,7 @@ def setUp(self): poll_sleep=POLL_SLEEP, location=TEST_LOCATION, ) + self.expected_airflow_version = 'v' + airflow.version.version.replace(".", "-").replace("+", "-") def test_init(self): """Test DataflowTemplateOperator instance is properly initialized.""" @@ -219,7 +222,7 @@ def test_exec(self, gcs_hook, dataflow_hook_mock, beam_hook_mock, mock_callback_ 'jobName': job_name, 'region': TEST_LOCATION, 'output': 'gs://test/output', - 'labels': {'foo': 'bar', 'airflow-version': 'v2-1-0-dev0'}, + 'labels': {'foo': 'bar', 'airflow-version': self.expected_airflow_version}, } start_java_mock.assert_called_once_with( @@ -260,7 +263,7 @@ def test_check_job_running_exec(self, gcs_hook, dataflow_mock, beam_hook_mock): 'jobName': JOB_NAME, 'region': TEST_LOCATION, 'output': 'gs://test/output', - 'labels': {'foo': 'bar', 'airflow-version': 'v2-1-0-dev0'}, + 'labels': {'foo': 'bar', 'airflow-version': self.expected_airflow_version}, } dataflow_running.assert_called_once_with(name=JOB_NAME, variables=variables) @@ -299,7 +302,7 @@ def set_is_job_dataflow_running_variables(*args, **kwargs): 'jobName': JOB_NAME, 'region': TEST_LOCATION, 'output': 'gs://test/output', - 'labels': {'foo': 'bar', 'airflow-version': 'v2-1-0-dev0'}, + 'labels': {'foo': 'bar', 'airflow-version': self.expected_airflow_version}, } self.assertEqual(expected_variables, is_job_dataflow_running_variables) job_name = dataflow_hook_mock.return_value.build_dataflow_job_name.return_value @@ -353,7 +356,7 @@ def set_is_job_dataflow_running_variables(*args, **kwargs): 'jobName': JOB_NAME, 'region': TEST_LOCATION, 'output': 'gs://test/output', - 'labels': {'foo': 'bar', 'airflow-version': 'v2-1-0-dev0'}, + 'labels': {'foo': 'bar', 'airflow-version': self.expected_airflow_version}, } self.assertEqual(expected_variables, is_job_dataflow_running_variables) job_name = dataflow_hook_mock.return_value.build_dataflow_job_name.return_value diff --git a/tests/providers/google/cloud/operators/test_dataproc.py b/tests/providers/google/cloud/operators/test_dataproc.py index fb2ceef1f98e7..764c225f1769a 100644 --- a/tests/providers/google/cloud/operators/test_dataproc.py +++ b/tests/providers/google/cloud/operators/test_dataproc.py @@ -19,19 +19,23 @@ import unittest from datetime import datetime from unittest import mock +from unittest.mock import MagicMock, Mock, call import pytest from google.api_core.exceptions import AlreadyExists, NotFound from google.api_core.retry import Retry from airflow import AirflowException +from airflow.models import DAG, DagBag, TaskInstance from airflow.providers.google.cloud.operators.dataproc import ( ClusterGenerator, + DataprocClusterLink, DataprocCreateClusterOperator, DataprocCreateWorkflowTemplateOperator, DataprocDeleteClusterOperator, DataprocInstantiateInlineWorkflowTemplateOperator, DataprocInstantiateWorkflowTemplateOperator, + DataprocJobLink, DataprocScaleClusterOperator, DataprocSubmitHadoopJobOperator, DataprocSubmitHiveJobOperator, @@ -42,7 +46,9 @@ DataprocSubmitSparkSqlJobOperator, DataprocUpdateClusterOperator, ) +from airflow.serialization.serialized_objects import SerializedDAG from airflow.version import version as airflow_version +from tests.test_utils.db import clear_db_runs, clear_db_xcom cluster_params = inspect.signature(ClusterGenerator.__init__).parameters @@ -171,12 +177,77 @@ }, "jobs": [{"step_id": "pig_job_1", "pig_job": {}}], } +TEST_DAG_ID = 'test-dataproc-operators' +DEFAULT_DATE = datetime(2020, 1, 1) +TEST_JOB_ID = "test-job" + +DATAPROC_JOB_LINK_EXPECTED = ( + f"https://console.cloud.google.com/dataproc/jobs/{TEST_JOB_ID}?" + f"region={GCP_LOCATION}&project={GCP_PROJECT}" +) +DATAPROC_CLUSTER_LINK_EXPECTED = ( + f"https://console.cloud.google.com/dataproc/clusters/{CLUSTER_NAME}/monitoring?" + f"region={GCP_LOCATION}&project={GCP_PROJECT}" +) +DATAPROC_JOB_CONF_EXPECTED = { + "job_id": TEST_JOB_ID, + "region": GCP_LOCATION, + "project_id": GCP_PROJECT, +} +DATAPROC_CLUSTER_CONF_EXPECTED = { + "cluster_name": CLUSTER_NAME, + "region": GCP_LOCATION, + "project_id": GCP_PROJECT, +} def assert_warning(msg: str, warnings): assert any(msg in str(w) for w in warnings) +class DataprocTestBase(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.dagbag = DagBag(dag_folder="/dev/null", include_examples=False) + cls.dag = DAG(TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}) + + def setUp(self): + self.mock_ti = MagicMock() + self.mock_context = {"ti": self.mock_ti} + self.extra_links_manager_mock = Mock() + self.extra_links_manager_mock.attach_mock(self.mock_ti, 'ti') + + def tearDown(self): + self.mock_ti = MagicMock() + self.mock_context = {"ti": self.mock_ti} + self.extra_links_manager_mock = Mock() + self.extra_links_manager_mock.attach_mock(self.mock_ti, 'ti') + + @classmethod + def tearDownClass(cls): + clear_db_runs() + clear_db_xcom() + + +class DataprocJobTestBase(DataprocTestBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.extra_links_expected_calls = [ + call.ti.xcom_push(execution_date=None, key='job_conf', value=DATAPROC_JOB_CONF_EXPECTED), + call.hook().wait_for_job(job_id=TEST_JOB_ID, location=GCP_LOCATION, project_id=GCP_PROJECT), + ] + + +class DataprocClusterTestBase(DataprocTestBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.extra_links_expected_calls_base = [ + call.ti.xcom_push(execution_date=None, key='cluster_conf', value=DATAPROC_CLUSTER_CONF_EXPECTED) + ] + + class TestsClusterGenerator(unittest.TestCase): def test_image_version(self): with pytest.raises(ValueError) as ctx: @@ -290,7 +361,7 @@ def test_build_with_custom_image_family(self): assert CONFIG_WITH_CUSTOM_IMAGE_FAMILY == cluster -class TestDataprocClusterCreateOperator(unittest.TestCase): +class TestDataprocClusterCreateOperator(DataprocClusterTestBase): def test_deprecation_warning(self): with pytest.warns(DeprecationWarning) as warnings: op = DataprocCreateClusterOperator( @@ -321,6 +392,23 @@ def test_deprecation_warning(self): @mock.patch(DATAPROC_PATH.format("Cluster.to_dict")) @mock.patch(DATAPROC_PATH.format("DataprocHook")) def test_execute(self, mock_hook, to_dict_mock): + self.extra_links_manager_mock.attach_mock(mock_hook, 'hook') + mock_hook.return_value.create_cluster.result.return_value = None + create_cluster_args = { + 'region': GCP_LOCATION, + 'project_id': GCP_PROJECT, + 'cluster_name': CLUSTER_NAME, + 'request_id': REQUEST_ID, + 'retry': RETRY, + 'timeout': TIMEOUT, + 'metadata': METADATA, + 'cluster_config': CONFIG, + 'labels': LABELS, + } + expected_calls = self.extra_links_expected_calls_base + [ + call.hook().create_cluster(**create_cluster_args), + ] + op = DataprocCreateClusterOperator( task_id=TASK_ID, region=GCP_LOCATION, @@ -335,20 +423,19 @@ def test_execute(self, mock_hook, to_dict_mock): metadata=METADATA, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) + op.execute(context=self.mock_context) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) - mock_hook.return_value.create_cluster.assert_called_once_with( - region=GCP_LOCATION, - project_id=GCP_PROJECT, - cluster_config=CONFIG, - labels=LABELS, - cluster_name=CLUSTER_NAME, - request_id=REQUEST_ID, - retry=RETRY, - timeout=TIMEOUT, - metadata=METADATA, - ) + mock_hook.return_value.create_cluster.assert_called_once_with(**create_cluster_args) + + # Test whether xcom push occurs before create cluster is called + self.extra_links_manager_mock.assert_has_calls(expected_calls, any_order=False) + to_dict_mock.assert_called_once_with(mock_hook().create_cluster().result()) + self.mock_ti.xcom_push.assert_called_once_with( + key="cluster_conf", + value=DATAPROC_CLUSTER_CONF_EXPECTED, + execution_date=None, + ) @mock.patch(DATAPROC_PATH.format("Cluster.to_dict")) @mock.patch(DATAPROC_PATH.format("DataprocHook")) @@ -369,7 +456,7 @@ def test_execute_if_cluster_exists(self, mock_hook, to_dict_mock): request_id=REQUEST_ID, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) + op.execute(context=self.mock_context) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) mock_hook.return_value.create_cluster.assert_called_once_with( region=GCP_LOCATION, @@ -411,7 +498,7 @@ def test_execute_if_cluster_exists_do_not_use(self, mock_hook): use_if_exists=False, ) with pytest.raises(AlreadyExists): - op.execute(context={}) + op.execute(context=self.mock_context) @mock.patch(DATAPROC_PATH.format("DataprocHook")) def test_execute_if_cluster_exists_in_error_state(self, mock_hook): @@ -435,7 +522,7 @@ def test_execute_if_cluster_exists_in_error_state(self, mock_hook): request_id=REQUEST_ID, ) with pytest.raises(AirflowException): - op.execute(context={}) + op.execute(context=self.mock_context) mock_hook.return_value.diagnose_cluster.assert_called_once_with( region=GCP_LOCATION, project_id=GCP_PROJECT, cluster_name=CLUSTER_NAME @@ -474,7 +561,7 @@ def test_execute_if_cluster_exists_in_deleting_state( gcp_conn_id=GCP_CONN_ID, ) with pytest.raises(AirflowException): - op.execute(context={}) + op.execute(context=self.mock_context) calls = [mock.call(mock_hook.return_value), mock.call(mock_hook.return_value)] mock_get_cluster.assert_has_calls(calls) @@ -483,8 +570,60 @@ def test_execute_if_cluster_exists_in_deleting_state( region=GCP_LOCATION, project_id=GCP_PROJECT, cluster_name=CLUSTER_NAME ) + def test_operator_extra_links(self): + op = DataprocCreateClusterOperator( + task_id=TASK_ID, + region=GCP_LOCATION, + project_id=GCP_PROJECT, + cluster_name=CLUSTER_NAME, + delete_on_error=True, + gcp_conn_id=GCP_CONN_ID, + dag=self.dag, + ) + + serialized_dag = SerializedDAG.to_dict(self.dag) + deserialized_dag = SerializedDAG.from_dict(serialized_dag) + deserialized_task = deserialized_dag.task_dict[TASK_ID] + + # Assert operator links for serialized DAG + self.assertEqual( + serialized_dag["dag"]["tasks"][0]["_operator_extra_links"], + [{"airflow.providers.google.cloud.operators.dataproc.DataprocClusterLink": {}}], + ) + + # Assert operator link types are preserved during deserialization + self.assertIsInstance(deserialized_task.operator_extra_links[0], DataprocClusterLink) + + ti = TaskInstance(task=op, execution_date=DEFAULT_DATE) + + # Assert operator link is empty when no XCom push occurred + self.assertEqual(op.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), "") + + # Assert operator link is empty for deserialized task when no XCom push occurred + self.assertEqual( + deserialized_task.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), + "", + ) + + ti.xcom_push(key="cluster_conf", value=DATAPROC_CLUSTER_CONF_EXPECTED) + + # Assert operator links are preserved in deserialized tasks after execution + self.assertEqual( + deserialized_task.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), + DATAPROC_CLUSTER_LINK_EXPECTED, + ) + + # Assert operator links after execution + self.assertEqual( + op.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), + DATAPROC_CLUSTER_LINK_EXPECTED, + ) + + # Check negative case + self.assertEqual(op.get_extra_links(datetime(2020, 7, 20), DataprocClusterLink.name), "") + -class TestDataprocClusterScaleOperator(unittest.TestCase): +class TestDataprocClusterScaleOperator(DataprocClusterTestBase): def test_deprecation_warning(self): with pytest.warns(DeprecationWarning) as warnings: DataprocScaleClusterOperator(task_id=TASK_ID, cluster_name=CLUSTER_NAME, project_id=GCP_PROJECT) @@ -492,9 +631,22 @@ def test_deprecation_warning(self): @mock.patch(DATAPROC_PATH.format("DataprocHook")) def test_execute(self, mock_hook): + self.extra_links_manager_mock.attach_mock(mock_hook, 'hook') + mock_hook.return_value.update_cluster.result.return_value = None cluster_update = { "config": {"worker_config": {"num_instances": 3}, "secondary_worker_config": {"num_instances": 4}} } + update_cluster_args = { + 'project_id': GCP_PROJECT, + 'location': GCP_LOCATION, + 'cluster_name': CLUSTER_NAME, + 'cluster': cluster_update, + 'graceful_decommission_timeout': {"seconds": 600}, + 'update_mask': UPDATE_MASK, + } + expected_calls = self.extra_links_expected_calls_base + [ + call.hook().update_cluster(**update_cluster_args) + ] op = DataprocScaleClusterOperator( task_id=TASK_ID, @@ -507,18 +659,73 @@ def test_execute(self, mock_hook): gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) - + op.execute(context=self.mock_context) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) - mock_hook.return_value.update_cluster.assert_called_once_with( - project_id=GCP_PROJECT, - location=GCP_LOCATION, + mock_hook.return_value.update_cluster.assert_called_once_with(**update_cluster_args) + + # Test whether xcom push occurs before cluster is updated + self.extra_links_manager_mock.assert_has_calls(expected_calls, any_order=False) + + self.mock_ti.xcom_push.assert_called_once_with( + key="cluster_conf", + value=DATAPROC_CLUSTER_CONF_EXPECTED, + execution_date=None, + ) + + def test_operator_extra_links(self): + op = DataprocScaleClusterOperator( + task_id=TASK_ID, cluster_name=CLUSTER_NAME, - cluster=cluster_update, - graceful_decommission_timeout={"seconds": 600}, - update_mask=UPDATE_MASK, + project_id=GCP_PROJECT, + region=GCP_LOCATION, + num_workers=3, + num_preemptible_workers=2, + graceful_decommission_timeout="2m", + gcp_conn_id=GCP_CONN_ID, + dag=self.dag, ) + serialized_dag = SerializedDAG.to_dict(self.dag) + deserialized_dag = SerializedDAG.from_dict(serialized_dag) + deserialized_task = deserialized_dag.task_dict[TASK_ID] + + # Assert operator links for serialized DAG + self.assertEqual( + serialized_dag["dag"]["tasks"][0]["_operator_extra_links"], + [{"airflow.providers.google.cloud.operators.dataproc.DataprocClusterLink": {}}], + ) + + # Assert operator link types are preserved during deserialization + self.assertIsInstance(deserialized_task.operator_extra_links[0], DataprocClusterLink) + + ti = TaskInstance(task=op, execution_date=DEFAULT_DATE) + + # Assert operator link is empty when no XCom push occurred + self.assertEqual(op.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), "") + + # Assert operator link is empty for deserialized task when no XCom push occurred + self.assertEqual( + deserialized_task.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), + "", + ) + + ti.xcom_push(key="cluster_conf", value=DATAPROC_CLUSTER_CONF_EXPECTED) + + # Assert operator links are preserved in deserialized tasks after execution + self.assertEqual( + deserialized_task.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), + DATAPROC_CLUSTER_LINK_EXPECTED, + ) + + # Assert operator links after execution + self.assertEqual( + op.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), + DATAPROC_CLUSTER_LINK_EXPECTED, + ) + + # Check negative case + self.assertEqual(op.get_extra_links(datetime(2020, 7, 20), DataprocClusterLink.name), "") + class TestDataprocClusterDeleteOperator(unittest.TestCase): @mock.patch(DATAPROC_PATH.format("DataprocHook")) @@ -549,13 +756,20 @@ def test_execute(self, mock_hook): ) -class TestDataprocSubmitJobOperator(unittest.TestCase): +class TestDataprocSubmitJobOperator(DataprocJobTestBase): @mock.patch(DATAPROC_PATH.format("DataprocHook")) def test_execute(self, mock_hook): + xcom_push_call = call.ti.xcom_push( + execution_date=None, key='job_conf', value=DATAPROC_JOB_CONF_EXPECTED + ) + wait_for_job_call = call.hook().wait_for_job( + job_id=TEST_JOB_ID, location=GCP_LOCATION, project_id=GCP_PROJECT, timeout=None + ) + job = {} - job_id = "job_id" mock_hook.return_value.wait_for_job.return_value = None - mock_hook.return_value.submit_job.return_value.reference.job_id = job_id + mock_hook.return_value.submit_job.return_value.reference.job_id = TEST_JOB_ID + self.extra_links_manager_mock.attach_mock(mock_hook, 'hook') op = DataprocSubmitJobOperator( task_id=TASK_ID, @@ -569,9 +783,17 @@ def test_execute(self, mock_hook): request_id=REQUEST_ID, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) + op.execute(context=self.mock_context) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) + + # Test whether xcom push occurs before polling for job + self.assertLess( + self.extra_links_manager_mock.mock_calls.index(xcom_push_call), + self.extra_links_manager_mock.mock_calls.index(wait_for_job_call), + msg='Xcom push for Job Link has to be done before polling for job status', + ) + mock_hook.return_value.submit_job.assert_called_once_with( project_id=GCP_PROJECT, location=GCP_LOCATION, @@ -582,15 +804,18 @@ def test_execute(self, mock_hook): metadata=METADATA, ) mock_hook.return_value.wait_for_job.assert_called_once_with( - job_id=job_id, project_id=GCP_PROJECT, location=GCP_LOCATION, timeout=None + job_id=TEST_JOB_ID, project_id=GCP_PROJECT, location=GCP_LOCATION, timeout=None + ) + + self.mock_ti.xcom_push.assert_called_once_with( + key="job_conf", value=DATAPROC_JOB_CONF_EXPECTED, execution_date=None ) @mock.patch(DATAPROC_PATH.format("DataprocHook")) def test_execute_async(self, mock_hook): job = {} - job_id = "job_id" mock_hook.return_value.wait_for_job.return_value = None - mock_hook.return_value.submit_job.return_value.reference.job_id = job_id + mock_hook.return_value.submit_job.return_value.reference.job_id = TEST_JOB_ID op = DataprocSubmitJobOperator( task_id=TASK_ID, @@ -605,7 +830,7 @@ def test_execute_async(self, mock_hook): request_id=REQUEST_ID, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) + op.execute(context=self.mock_context) mock_hook.assert_called_once_with( gcp_conn_id=GCP_CONN_ID, @@ -622,6 +847,10 @@ def test_execute_async(self, mock_hook): ) mock_hook.return_value.wait_for_job.assert_not_called() + self.mock_ti.xcom_push.assert_called_once_with( + key="job_conf", value=DATAPROC_JOB_CONF_EXPECTED, execution_date=None + ) + @mock.patch(DATAPROC_PATH.format("DataprocHook")) def test_on_kill(self, mock_hook): job = {} @@ -642,7 +871,7 @@ def test_on_kill(self, mock_hook): impersonation_chain=IMPERSONATION_CHAIN, cancel_on_kill=False, ) - op.execute(context={}) + op.execute(context=self.mock_context) op.on_kill() mock_hook.return_value.cancel_job.assert_not_called() @@ -653,10 +882,77 @@ def test_on_kill(self, mock_hook): project_id=GCP_PROJECT, location=GCP_LOCATION, job_id=job_id ) + @mock.patch(DATAPROC_PATH.format("DataprocHook")) + def test_operator_extra_links(self, mock_hook): + mock_hook.return_value.project_id = GCP_PROJECT + op = DataprocSubmitJobOperator( + task_id=TASK_ID, + location=GCP_LOCATION, + project_id=GCP_PROJECT, + job={}, + gcp_conn_id=GCP_CONN_ID, + dag=self.dag, + ) + + serialized_dag = SerializedDAG.to_dict(self.dag) + deserialized_dag = SerializedDAG.from_dict(serialized_dag) + deserialized_task = deserialized_dag.task_dict[TASK_ID] + + # Assert operator links for serialized_dag + self.assertEqual( + serialized_dag["dag"]["tasks"][0]["_operator_extra_links"], + [{"airflow.providers.google.cloud.operators.dataproc.DataprocJobLink": {}}], + ) + + # Assert operator link types are preserved during deserialization + self.assertIsInstance(deserialized_task.operator_extra_links[0], DataprocJobLink) + + ti = TaskInstance(task=op, execution_date=DEFAULT_DATE) + + # Assert operator link is empty when no XCom push occurred + self.assertEqual(op.get_extra_links(DEFAULT_DATE, DataprocJobLink.name), "") + + # Assert operator link is empty for deserialized task when no XCom push occurred + self.assertEqual(deserialized_task.get_extra_links(DEFAULT_DATE, DataprocJobLink.name), "") + + ti.xcom_push(key="job_conf", value=DATAPROC_JOB_CONF_EXPECTED) -class TestDataprocUpdateClusterOperator(unittest.TestCase): + # Assert operator links are preserved in deserialized tasks + self.assertEqual( + deserialized_task.get_extra_links(DEFAULT_DATE, DataprocJobLink.name), + DATAPROC_JOB_LINK_EXPECTED, + ) + # Assert operator links after execution + self.assertEqual( + op.get_extra_links(DEFAULT_DATE, DataprocJobLink.name), + DATAPROC_JOB_LINK_EXPECTED, + ) + # Check for negative case + self.assertEqual(op.get_extra_links(datetime(2020, 7, 20), DataprocJobLink.name), "") + + +class TestDataprocUpdateClusterOperator(DataprocClusterTestBase): @mock.patch(DATAPROC_PATH.format("DataprocHook")) def test_execute(self, mock_hook): + self.extra_links_manager_mock.attach_mock(mock_hook, 'hook') + mock_hook.return_value.update_cluster.result.return_value = None + cluster_decommission_timeout = {"graceful_decommission_timeout": "600s"} + update_cluster_args = { + 'location': GCP_LOCATION, + 'project_id': GCP_PROJECT, + 'cluster_name': CLUSTER_NAME, + 'cluster': CLUSTER, + 'update_mask': UPDATE_MASK, + 'graceful_decommission_timeout': cluster_decommission_timeout, + 'request_id': REQUEST_ID, + 'retry': RETRY, + 'timeout': TIMEOUT, + 'metadata': METADATA, + } + expected_calls = self.extra_links_expected_calls_base + [ + call.hook().update_cluster(**update_cluster_args) + ] + op = DataprocUpdateClusterOperator( task_id=TASK_ID, location=GCP_LOCATION, @@ -664,7 +960,7 @@ def test_execute(self, mock_hook): cluster=CLUSTER, update_mask=UPDATE_MASK, request_id=REQUEST_ID, - graceful_decommission_timeout={"graceful_decommission_timeout": "600s"}, + graceful_decommission_timeout=cluster_decommission_timeout, project_id=GCP_PROJECT, gcp_conn_id=GCP_CONN_ID, retry=RETRY, @@ -672,21 +968,71 @@ def test_execute(self, mock_hook): metadata=METADATA, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) + op.execute(context=self.mock_context) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) - mock_hook.return_value.update_cluster.assert_called_once_with( + mock_hook.return_value.update_cluster.assert_called_once_with(**update_cluster_args) + + # Test whether the xcom push happens before updating the cluster + self.extra_links_manager_mock.assert_has_calls(expected_calls, any_order=False) + + self.mock_ti.xcom_push.assert_called_once_with( + key="cluster_conf", + value=DATAPROC_CLUSTER_CONF_EXPECTED, + execution_date=None, + ) + + def test_operator_extra_links(self): + op = DataprocUpdateClusterOperator( + task_id=TASK_ID, location=GCP_LOCATION, - project_id=GCP_PROJECT, cluster_name=CLUSTER_NAME, cluster=CLUSTER, update_mask=UPDATE_MASK, graceful_decommission_timeout={"graceful_decommission_timeout": "600s"}, - request_id=REQUEST_ID, - retry=RETRY, - timeout=TIMEOUT, - metadata=METADATA, + project_id=GCP_PROJECT, + gcp_conn_id=GCP_CONN_ID, + dag=self.dag, ) + serialized_dag = SerializedDAG.to_dict(self.dag) + deserialized_dag = SerializedDAG.from_dict(serialized_dag) + deserialized_task = deserialized_dag.task_dict[TASK_ID] + + # Assert operator links for serialized_dag + self.assertEqual( + serialized_dag["dag"]["tasks"][0]["_operator_extra_links"], + [{"airflow.providers.google.cloud.operators.dataproc.DataprocClusterLink": {}}], + ) + + # Assert operator link types are preserved during deserialization + self.assertIsInstance(deserialized_task.operator_extra_links[0], DataprocClusterLink) + + ti = TaskInstance(task=op, execution_date=DEFAULT_DATE) + + # Assert operator link is empty when no XCom push occurred + self.assertEqual(op.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), "") + + # Assert operator link is empty for deserialized task when no XCom push occurred + self.assertEqual( + deserialized_task.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), + "", + ) + + ti.xcom_push(key="cluster_conf", value=DATAPROC_CLUSTER_CONF_EXPECTED) + + # Assert operator links are preserved in deserialized tasks + self.assertEqual( + deserialized_task.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), + DATAPROC_CLUSTER_LINK_EXPECTED, + ) + # Assert operator links after execution + self.assertEqual( + op.get_extra_links(DEFAULT_DATE, DataprocClusterLink.name), + DATAPROC_CLUSTER_LINK_EXPECTED, + ) + # Check for negative case + self.assertEqual(op.get_extra_links(datetime(2020, 7, 20), DataprocClusterLink.name), "") + class TestDataprocWorkflowTemplateInstantiateOperator(unittest.TestCase): @mock.patch(DATAPROC_PATH.format("DataprocHook")) @@ -787,7 +1133,7 @@ def test_execute(self, mock_hook, mock_uuid): variables=self.variables, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) + op.execute(context=MagicMock()) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) mock_hook.return_value.submit_job.assert_called_once_with( project_id=GCP_PROJECT, job=self.job, location=GCP_LOCATION @@ -846,7 +1192,7 @@ def test_execute(self, mock_hook, mock_uuid): variables=self.variables, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) + op.execute(context=MagicMock()) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) mock_hook.return_value.submit_job.assert_called_once_with( project_id=GCP_PROJECT, job=self.job, location=GCP_LOCATION @@ -911,7 +1257,7 @@ def test_execute(self, mock_hook, mock_uuid): variables=self.variables, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) + op.execute(context=MagicMock()) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) mock_hook.return_value.submit_job.assert_called_once_with( project_id=GCP_PROJECT, job=self.job, location=GCP_LOCATION @@ -937,7 +1283,7 @@ def test_execute_override_project_id(self, mock_hook, mock_uuid): variables=self.variables, impersonation_chain=IMPERSONATION_CHAIN, ) - op.execute(context={}) + op.execute(context=MagicMock()) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) mock_hook.return_value.submit_job.assert_called_once_with( project_id="other-project", job=self.other_project_job, location=GCP_LOCATION @@ -963,12 +1309,14 @@ def test_builder(self, mock_hook, mock_uuid): assert self.job == job -class TestDataProcSparkOperator(unittest.TestCase): +class TestDataProcSparkOperator(DataprocJobTestBase): main_class = "org.apache.spark.examples.SparkPi" jars = ["file:///usr/lib/spark/examples/jars/spark-examples.jar"] - job_id = "uuid_id" job = { - "reference": {"project_id": GCP_PROJECT, "job_id": "{{task.task_id}}_{{ds_nodash}}_" + job_id}, + "reference": { + "project_id": GCP_PROJECT, + "job_id": "{{task.task_id}}_{{ds_nodash}}_" + TEST_JOB_ID, + }, "placement": {"cluster_name": "cluster-1"}, "labels": {"airflow-version": AIRFLOW_VERSION}, "spark_job": {"jar_file_uris": jars, "main_class": main_class}, @@ -985,9 +1333,11 @@ def test_deprecation_warning(self, mock_hook): @mock.patch(DATAPROC_PATH.format("uuid.uuid4")) @mock.patch(DATAPROC_PATH.format("DataprocHook")) def test_execute(self, mock_hook, mock_uuid): - mock_uuid.return_value = self.job_id + mock_uuid.return_value = TEST_JOB_ID mock_hook.return_value.project_id = GCP_PROJECT - mock_uuid.return_value = self.job_id + mock_uuid.return_value = TEST_JOB_ID + mock_hook.return_value.submit_job.return_value.reference.job_id = TEST_JOB_ID + self.extra_links_manager_mock.attach_mock(mock_hook, 'hook') op = DataprocSubmitSparkJobOperator( task_id=TASK_ID, @@ -999,6 +1349,68 @@ def test_execute(self, mock_hook, mock_uuid): job = op.generate_job() assert self.job == job + op.execute(context=self.mock_context) + self.mock_ti.xcom_push.assert_called_once_with( + key="job_conf", value=DATAPROC_JOB_CONF_EXPECTED, execution_date=None + ) + + # Test whether xcom push occurs before polling for job + self.extra_links_manager_mock.assert_has_calls(self.extra_links_expected_calls, any_order=False) + + @mock.patch(DATAPROC_PATH.format("DataprocHook")) + def test_operator_extra_links(self, mock_hook): + mock_hook.return_value.project_id = GCP_PROJECT + + op = DataprocSubmitSparkJobOperator( + task_id=TASK_ID, + region=GCP_LOCATION, + gcp_conn_id=GCP_CONN_ID, + main_class=self.main_class, + dataproc_jars=self.jars, + dag=self.dag, + ) + + serialized_dag = SerializedDAG.to_dict(self.dag) + deserialized_dag = SerializedDAG.from_dict(serialized_dag) + deserialized_task = deserialized_dag.task_dict[TASK_ID] + + # Assert operator links for serialized DAG + self.assertEqual( + serialized_dag["dag"]["tasks"][0]["_operator_extra_links"], + [{"airflow.providers.google.cloud.operators.dataproc.DataprocJobLink": {}}], + ) + + # Assert operator link types are preserved during deserialization + self.assertIsInstance(deserialized_task.operator_extra_links[0], DataprocJobLink) + + ti = TaskInstance(task=op, execution_date=DEFAULT_DATE) + + # Assert operator link is empty when no XCom push occurred + self.assertEqual(op.get_extra_links(DEFAULT_DATE, DataprocJobLink.name), "") + + # Assert operator link is empty for deserialized task when no XCom push occurred + self.assertEqual(deserialized_task.get_extra_links(DEFAULT_DATE, DataprocJobLink.name), "") + + ti.xcom_push(key="job_conf", value=DATAPROC_JOB_CONF_EXPECTED) + + # Assert operator links after task execution + self.assertEqual( + op.get_extra_links(DEFAULT_DATE, DataprocJobLink.name), + DATAPROC_JOB_LINK_EXPECTED, + ) + + # Assert operator links are preserved in deserialized tasks + self.assertEqual( + deserialized_task.get_extra_links(DEFAULT_DATE, DataprocJobLink.name), + DATAPROC_JOB_LINK_EXPECTED, + ) + + # Assert for negative case + self.assertEqual( + deserialized_task.get_extra_links(datetime(2020, 7, 20), DataprocJobLink.name), + "", + ) + class TestDataProcHadoopOperator(unittest.TestCase): args = ["wordcount", "gs://pub/shakespeare/rose.txt"] diff --git a/tests/providers/google/cloud/operators/test_tasks.py b/tests/providers/google/cloud/operators/test_tasks.py index 58523bcd66700..585e2612a7e85 100644 --- a/tests/providers/google/cloud/operators/test_tasks.py +++ b/tests/providers/google/cloud/operators/test_tasks.py @@ -151,9 +151,8 @@ def test_delete_queue(self, mock_hook): mock_hook.return_value.delete_queue.return_value = None operator = CloudTasksQueueDeleteOperator(location=LOCATION, queue_name=QUEUE_ID, task_id="id") - result = operator.execute(context=None) + operator.execute(context=None) - assert result is None mock_hook.assert_called_once_with( gcp_conn_id=GCP_CONN_ID, impersonation_chain=None, @@ -350,9 +349,8 @@ def test_delete_task(self, mock_hook): location=LOCATION, queue_name=QUEUE_ID, task_name=TASK_NAME, task_id="id" ) - result = operator.execute(context=None) + operator.execute(context=None) - assert result is None mock_hook.assert_called_once_with( gcp_conn_id=GCP_CONN_ID, impersonation_chain=None, diff --git a/tests/providers/google/cloud/transfers/test_presto_to_gcs_system.py b/tests/providers/google/cloud/transfers/test_presto_to_gcs_system.py index b630b2e8044e2..ba0bf8a9428b0 100644 --- a/tests/providers/google/cloud/transfers/test_presto_to_gcs_system.py +++ b/tests/providers/google/cloud/transfers/test_presto_to_gcs_system.py @@ -22,21 +22,10 @@ from airflow.models import Connection from airflow.providers.presto.hooks.presto import PrestoHook +from airflow.utils.session import create_session from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY, GCP_GCS_KEY from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context -try: - from airflow.utils.session import create_session -except ImportError: - # This is a hack to import create_session from old destination and - # fool the pre-commit check that looks for old imports... - # TODO remove this once we don't need to test this on 1.10 - import importlib - - db_module = importlib.import_module("airflow.utils.db") - create_session = getattr(db_module, "create_session") - - GCS_BUCKET = os.environ.get("GCP_PRESTO_TO_GCS_BUCKET_NAME", "test-presto-to-gcs-bucket") DATASET_NAME = os.environ.get("GCP_PRESTO_TO_GCS_DATASET_NAME", "test_presto_to_gcs_dataset") diff --git a/tests/providers/google/cloud/transfers/test_trino_to_gcs_system.py b/tests/providers/google/cloud/transfers/test_trino_to_gcs_system.py index 00d5716556183..66d505fba8330 100644 --- a/tests/providers/google/cloud/transfers/test_trino_to_gcs_system.py +++ b/tests/providers/google/cloud/transfers/test_trino_to_gcs_system.py @@ -22,21 +22,10 @@ from airflow.models import Connection from airflow.providers.trino.hooks.trino import TrinoHook +from airflow.utils.session import create_session from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY, GCP_GCS_KEY from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context -try: - from airflow.utils.session import create_session -except ImportError: - # This is a hack to import create_session from old destination and - # fool the pre-commit check that looks for old imports... - # TODO remove this once we don't need to test this on 1.10 - import importlib - - db_module = importlib.import_module("airflow.utils.db") - create_session = getattr(db_module, "create_session") - - GCS_BUCKET = os.environ.get("GCP_TRINO_TO_GCS_BUCKET_NAME", "test-trino-to-gcs-bucket") DATASET_NAME = os.environ.get("GCP_TRINO_TO_GCS_DATASET_NAME", "test_trino_to_gcs_dataset") diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 816adc3686fa8..825847b982156 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -16,7 +16,9 @@ # specific language governing permissions and limitations # under the License. import json +import os import unittest +from collections import OrderedDict from unittest import mock import pytest @@ -279,5 +281,77 @@ def match_obj1(request): # will raise NoMockAddress exception if obj1 != request.json() HttpHook(method=method).run('v1/test', json=obj1) + @mock.patch('airflow.providers.http.hooks.http.requests.Session.send') + def test_verify_set_to_true_by_default(self, mock_session_send): + with mock.patch( + 'airflow.hooks.base.BaseHook.get_connection', side_effect=get_airflow_connection_with_port + ): + self.get_hook.run('/some/endpoint') + mock_session_send.assert_called_once_with( + mock.ANY, + allow_redirects=True, + cert=None, + proxies=OrderedDict(), + stream=False, + timeout=None, + verify=True, + ) + + @mock.patch('airflow.providers.http.hooks.http.requests.Session.send') + @mock.patch.dict(os.environ, {"REQUESTS_CA_BUNDLE": "/tmp/test.crt"}) + def test_requests_ca_bundle_env_var(self, mock_session_send): + with mock.patch( + 'airflow.hooks.base.BaseHook.get_connection', side_effect=get_airflow_connection_with_port + ): + + self.get_hook.run('/some/endpoint') + + mock_session_send.assert_called_once_with( + mock.ANY, + allow_redirects=True, + cert=None, + proxies=OrderedDict(), + stream=False, + timeout=None, + verify='/tmp/test.crt', + ) + + @mock.patch('airflow.providers.http.hooks.http.requests.Session.send') + @mock.patch.dict(os.environ, {"REQUESTS_CA_BUNDLE": "/tmp/test.crt"}) + def test_verify_respects_requests_ca_bundle_env_var(self, mock_session_send): + with mock.patch( + 'airflow.hooks.base.BaseHook.get_connection', side_effect=get_airflow_connection_with_port + ): + + self.get_hook.run('/some/endpoint', extra_options={'verify': True}) + + mock_session_send.assert_called_once_with( + mock.ANY, + allow_redirects=True, + cert=None, + proxies=OrderedDict(), + stream=False, + timeout=None, + verify='/tmp/test.crt', + ) + + @mock.patch('airflow.providers.http.hooks.http.requests.Session.send') + @mock.patch.dict(os.environ, {"REQUESTS_CA_BUNDLE": "/tmp/test.crt"}) + def test_verify_false_parameter_overwrites_set_requests_ca_bundle_env_var(self, mock_session_send): + with mock.patch( + 'airflow.hooks.base.BaseHook.get_connection', side_effect=get_airflow_connection_with_port + ): + self.get_hook.run('/some/endpoint', extra_options={'verify': False}) + + mock_session_send.assert_called_once_with( + mock.ANY, + allow_redirects=True, + cert=None, + proxies=OrderedDict(), + stream=False, + timeout=None, + verify=False, + ) + send_email_test = mock.Mock() diff --git a/tests/providers/http/sensors/test_http.py b/tests/providers/http/sensors/test_http.py index 569d2357d8627..23ac2fdf2f69f 100644 --- a/tests/providers/http/sensors/test_http.py +++ b/tests/providers/http/sensors/test_http.py @@ -150,7 +150,7 @@ def resp_check(_): response = requests.Response() response.status_code = 404 response.reason = 'Not Found' - response._content = b'This endpoint doesnt exist' + response._content = b"This endpoint doesn't exist" mock_session_send.return_value = response task = HttpSensor( @@ -172,17 +172,17 @@ def resp_check(_): assert mock_errors.called calls = [ mock.call('HTTP error: %s', 'Not Found'), - mock.call('This endpoint doesnt exist'), + mock.call("This endpoint doesn't exist"), mock.call('HTTP error: %s', 'Not Found'), - mock.call('This endpoint doesnt exist'), + mock.call("This endpoint doesn't exist"), mock.call('HTTP error: %s', 'Not Found'), - mock.call('This endpoint doesnt exist'), + mock.call("This endpoint doesn't exist"), mock.call('HTTP error: %s', 'Not Found'), - mock.call('This endpoint doesnt exist'), + mock.call("This endpoint doesn't exist"), mock.call('HTTP error: %s', 'Not Found'), - mock.call('This endpoint doesnt exist'), + mock.call("This endpoint doesn't exist"), mock.call('HTTP error: %s', 'Not Found'), - mock.call('This endpoint doesnt exist'), + mock.call("This endpoint doesn't exist"), ] mock_errors.assert_has_calls(calls) @@ -201,6 +201,9 @@ def prepare_request(self, request): self.response._content += ('/' + request.params['date']).encode('ascii', 'ignore') return self.response + def merge_environment_settings(self, _url, **kwargs): + return kwargs + class TestHttpOpSensor(unittest.TestCase): def setUp(self): diff --git a/tests/providers/microsoft/azure/hooks/test_wasb.py b/tests/providers/microsoft/azure/hooks/test_wasb.py index 65c2401915c8f..f617f7239bdc3 100644 --- a/tests/providers/microsoft/azure/hooks/test_wasb.py +++ b/tests/providers/microsoft/azure/hooks/test_wasb.py @@ -237,7 +237,7 @@ def test_delete_multiple_blobs(self, mock_check, mock_get_blobslist, mock_delete mock_get_blobslist.return_value = ['blob_prefix/blob1', 'blob_prefix/blob2'] hook = WasbHook(wasb_conn_id=self.shared_key_conn_id) hook.delete_file('container', 'blob_prefix', is_prefix=True) - mock_get_blobslist.assert_called_once_with('container', prefix='blob_prefix') + mock_get_blobslist.assert_called_once_with('container', prefix='blob_prefix', delimiter='') mock_delete_blobs.assert_any_call( 'container', 'blob_prefix/blob1', diff --git a/tests/providers/oracle/hooks/test_oracle.py b/tests/providers/oracle/hooks/test_oracle.py index 7e427ddc82335..d22a0b4858bd0 100644 --- a/tests/providers/oracle/hooks/test_oracle.py +++ b/tests/providers/oracle/hooks/test_oracle.py @@ -39,7 +39,9 @@ class TestOracleHookConn(unittest.TestCase): def setUp(self): super().setUp() - self.connection = Connection(login='login', password='password', host='host', port=1521) + self.connection = Connection( + login='login', password='password', host='host', schema='schema', port=1521 + ) self.db_hook = OracleHook() self.db_hook.get_connection = mock.Mock() @@ -53,28 +55,39 @@ def test_get_conn_host(self, mock_connect): assert args == () assert kwargs['user'] == 'login' assert kwargs['password'] == 'password' - assert kwargs['dsn'] == 'host' + assert kwargs['dsn'] == 'host:1521/schema' + + @mock.patch('airflow.providers.oracle.hooks.oracle.cx_Oracle.connect') + def test_get_conn_host_alternative_port(self, mock_connect): + self.connection.port = 1522 + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert args == () + assert kwargs['user'] == 'login' + assert kwargs['password'] == 'password' + assert kwargs['dsn'] == 'host:1522/schema' @mock.patch('airflow.providers.oracle.hooks.oracle.cx_Oracle.connect') def test_get_conn_sid(self, mock_connect): - dsn_sid = {'dsn': 'dsn', 'sid': 'sid'} + dsn_sid = {'dsn': 'ignored', 'sid': 'sid'} self.connection.extra = json.dumps(dsn_sid) self.db_hook.get_conn() assert mock_connect.call_count == 1 args, kwargs = mock_connect.call_args assert args == () - assert kwargs['dsn'] == cx_Oracle.makedsn(dsn_sid['dsn'], self.connection.port, dsn_sid['sid']) + assert kwargs['dsn'] == cx_Oracle.makedsn("host", self.connection.port, dsn_sid['sid']) @mock.patch('airflow.providers.oracle.hooks.oracle.cx_Oracle.connect') def test_get_conn_service_name(self, mock_connect): - dsn_service_name = {'dsn': 'dsn', 'service_name': 'service_name'} + dsn_service_name = {'dsn': 'ignored', 'service_name': 'service_name'} self.connection.extra = json.dumps(dsn_service_name) self.db_hook.get_conn() assert mock_connect.call_count == 1 args, kwargs = mock_connect.call_args assert args == () assert kwargs['dsn'] == cx_Oracle.makedsn( - dsn_service_name['dsn'], self.connection.port, service_name=dsn_service_name['service_name'] + "host", self.connection.port, service_name=dsn_service_name['service_name'] ) @mock.patch('airflow.providers.oracle.hooks.oracle.cx_Oracle.connect') diff --git a/tests/sensors/test_timeout_sensor.py b/tests/sensors/test_timeout_sensor.py index 63b42f42a2ef9..894bf8ddfa698 100644 --- a/tests/sensors/test_timeout_sensor.py +++ b/tests/sensors/test_timeout_sensor.py @@ -25,7 +25,6 @@ from airflow.models.dag import DAG from airflow.sensors.base import BaseSensorOperator from airflow.utils import timezone -from airflow.utils.decorators import apply_defaults from airflow.utils.timezone import datetime DEFAULT_DATE = datetime(2015, 1, 1) @@ -40,7 +39,6 @@ class TimeoutTestSensor(BaseSensorOperator): :type return_value: any """ - @apply_defaults def __init__(self, return_value=False, **kwargs): self.return_value = return_value super().__init__(**kwargs) diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index 6a186c5e57f3f..c1ce92e138faa 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -145,6 +145,7 @@ }, }, "edge_info": {}, + "dag_dependencies": [], }, } @@ -817,7 +818,13 @@ def test_dag_serialized_fields_with_schema(self): dag_schema: dict = load_dag_schema_dict()["definitions"]["dag"]["properties"] # The parameters we add manually in Serialization needs to be ignored - ignored_keys: set = {"is_subdag", "tasks", "has_on_success_callback", "has_on_failure_callback"} + ignored_keys: set = { + "is_subdag", + "tasks", + "has_on_success_callback", + "has_on_failure_callback", + "dag_dependencies", + } dag_params: set = set(dag_schema.keys()) - ignored_keys assert set(DAG.get_serialized_fields()) == dag_params diff --git a/tests/test_utils/mock_operators.py b/tests/test_utils/mock_operators.py index 989b984975970..67176343b893c 100644 --- a/tests/test_utils/mock_operators.py +++ b/tests/test_utils/mock_operators.py @@ -23,7 +23,6 @@ from airflow.models import TaskInstance from airflow.models.baseoperator import BaseOperator, BaseOperatorLink from airflow.providers.apache.hive.operators.hive import HiveOperator -from airflow.utils.decorators import apply_defaults # Namedtuple for testing purposes @@ -37,7 +36,6 @@ class MockOperator(BaseOperator): template_fields = ("arg1", "arg2") - @apply_defaults def __init__(self, arg1: str = "", arg2: str = "", **kwargs): super().__init__(**kwargs) self.arg1 = arg1 @@ -117,7 +115,6 @@ def operator_extra_links(self): return (CustomOpLink(),) return (CustomBaseIndexOpLink(i) for i, _ in enumerate(self.bash_command)) - @apply_defaults def __init__(self, bash_command=None, **kwargs): super().__init__(**kwargs) self.bash_command = bash_command @@ -169,9 +166,8 @@ def __init__(self, *args, **kwargs): class DeprecatedOperator(BaseOperator): - @apply_defaults def __init__(self, **kwargs): - warnings.warn("This operator is deprecated.", DeprecationWarning, stacklevel=4) + warnings.warn("This operator is deprecated.", DeprecationWarning, stacklevel=2) super().__init__(**kwargs) def execute(self, context): diff --git a/tests/test_utils/perf/dags/elastic_dag.py b/tests/test_utils/perf/dags/elastic_dag.py index 10c93f1966ee0..51db138aeeb1b 100644 --- a/tests/test_utils/perf/dags/elastic_dag.py +++ b/tests/test_utils/perf/dags/elastic_dag.py @@ -27,7 +27,7 @@ # DAG File used in performance tests. Its shape can be configured by environment variables. RE_TIME_DELTA = re.compile( - r"^((?P[\.\d]+?)d)?((?P[\.\d]+?)h)?((?P[\.\d]+?)m)?((?P[\.\d]+?)s)?$" + r"^((?P[.\d]+?)d)?((?P[.\d]+?)h)?((?P[.\d]+?)m)?((?P[.\d]+?)s)?$" ) diff --git a/tests/test_utils/perf/scheduler_dag_execution_timing.py b/tests/test_utils/perf/scheduler_dag_execution_timing.py index 87ea15f5a05c7..680b2ead045ec 100755 --- a/tests/test_utils/perf/scheduler_dag_execution_timing.py +++ b/tests/test_utils/perf/scheduler_dag_execution_timing.py @@ -103,11 +103,7 @@ def get_executor_under_test(dotted_path): from airflow.executors.executor_loader import ExecutorLoader if dotted_path == "MockExecutor": - try: - # Run against master and 1.10.x releases - from tests.test_utils.mock_executor import MockExecutor as executor - except ImportError: - from tests.executors.test_executor import TestExecutor as executor + from tests.test_utils.mock_executor import MockExecutor as executor else: executor = ExecutorLoader.load_executor(dotted_path) @@ -289,7 +285,7 @@ def main(num_runs, repeat, pre_create_dag_runs, executor_class, dag_ids): times = [] - # Need a lambda to refer to the _latest_ value fo scheduler_job, not just + # Need a lambda to refer to the _latest_ value for scheduler_job, not just # the initial one code_to_test = lambda: scheduler_job.run() # pylint: disable=unnecessary-lambda diff --git a/tests/test_utils/www.py b/tests/test_utils/www.py new file mode 100644 index 0000000000000..d8118dfdcd8e3 --- /dev/null +++ b/tests/test_utils/www.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from unittest import mock + + +def client_with_login(app, **kwargs): + patch_path = "flask_appbuilder.security.manager.check_password_hash" + with mock.patch(patch_path) as check_password_hash: + check_password_hash.return_value = True + client = app.test_client() + resp = client.post("/login/", data=kwargs) + assert resp.status_code == 302 + return client + + +def check_content_in_response(text, resp, resp_code=200): + resp_html = resp.data.decode('utf-8') + assert resp_code == resp.status_code + if isinstance(text, list): + for line in text: + assert line in resp_html + else: + assert text in resp_html + + +def check_content_not_in_response(text, resp, resp_code=200): + resp_html = resp.data.decode('utf-8') + assert resp_code == resp.status_code + if isinstance(text, list): + for line in text: + assert line not in resp_html + else: + assert text not in resp_html diff --git a/tests/utils/log/test_secrets_masker.py b/tests/utils/log/test_secrets_masker.py new file mode 100644 index 0000000000000..ba88b87a8b009 --- /dev/null +++ b/tests/utils/log/test_secrets_masker.py @@ -0,0 +1,236 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import inspect +import logging +import logging.config +import os +import textwrap + +import pytest + +from airflow.utils.log.secrets_masker import SecretsMasker, should_hide_value_for_key +from tests.test_utils.config import conf_vars + + +@pytest.fixture +def logger(caplog): + logging.config.dictConfig( + { + 'version': 1, + 'handlers': { + __name__: { + # Reset later + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stdout', + } + }, + 'loggers': { + __name__: { + 'handlers': [__name__], + 'level': logging.INFO, + 'propagate': False, + } + }, + 'disable_existing_loggers': False, + } + ) + formatter = ShortExcFormatter("%(levelname)s %(message)s") + logger = logging.getLogger(__name__) + + caplog.handler.setFormatter(formatter) + logger.handlers = [caplog.handler] + filt = SecretsMasker() + logger.addFilter(filt) + + filt.add_mask('password') + + return logger + + +class TestSecretsMasker: + def test_message(self, logger, caplog): + logger.info("XpasswordY") + + assert caplog.text == "INFO X***Y\n" + + def test_args(self, logger, caplog): + logger.info("Cannot connect to %s", "user:password") + + assert caplog.text == "INFO Cannot connect to user:***\n" + + def test_extra(self, logger, caplog): + logger.handlers[0].formatter = ShortExcFormatter("%(levelname)s %(message)s %(conn)s") + logger.info("Cannot connect", extra={'conn': "user:password"}) + + assert caplog.text == "INFO Cannot connect user:***\n" + + def test_exception(self, logger, caplog): + try: + conn = "user:password" + raise RuntimeError("Cannot connect to " + conn) + except RuntimeError: + logger.exception("Err") + + line = lineno() - 4 + + assert caplog.text == textwrap.dedent( + f"""\ + ERROR Err + Traceback (most recent call last): + File ".../test_secrets_masker.py", line {line}, in test_exception + raise RuntimeError("Cannot connect to " + conn) + RuntimeError: Cannot connect to user:*** + """ + ) + + @pytest.mark.xfail(reason="Cannot filter secrets in traceback source") + def test_exc_tb(self, logger, caplog): + """ + Show it is not possible to filter secrets in the source. + + It is not possible to (regularly/reliably) filter out secrets that + appear directly in the source code. This is because the formatting of + exc_info is not done in the filter, it is done after the filter is + called, and fixing this "properly" is hard/impossible. + + (It would likely need to construct a custom traceback that changed the + source. I have no idead if that is even possible) + + This test illustrates that, but ix marked xfail in case someone wants to + fix this later. + """ + try: + raise RuntimeError("Cannot connect to user:password") + except RuntimeError: + logger.exception("Err") + + line = lineno() - 4 + + assert caplog.text == textwrap.dedent( + f"""\ + ERROR Err + Traceback (most recent call last): + File ".../test_secrets_masker.py", line {line}, in test_exc_tb + raise RuntimeError("Cannot connect to user:***) + RuntimeError: Cannot connect to user:*** + """ + ) + + @pytest.mark.parametrize( + ("name", "value", "expected_mask"), + [ + (None, "secret", {"secret"}), + ("apikey", "secret", {"secret"}), + # the value for "apikey", and "password" should end up masked + (None, {"apikey": "secret", "other": {"val": "innocent", "password": "foo"}}, {"secret", "foo"}), + (None, ["secret", "other"], {"secret", "other"}), + # When the "sensitive value" is a dict, don't mask anything + # (Or should this be mask _everything_ under it ? + ("api_key", {"other": "innoent"}, set()), + ], + ) + def test_mask_secret(self, name, value, expected_mask): + filt = SecretsMasker() + filt.add_mask(value, name) + + assert filt.patterns == expected_mask + + @pytest.mark.parametrize( + ("patterns", "name", "value", "expected"), + [ + ({"secret"}, None, "secret", "***"), + ( + {"secret", "foo"}, + None, + {"apikey": "secret", "other": {"val": "innocent", "password": "foo"}}, + {"apikey": "***", "other": {"val": "innocent", "password": "***"}}, + ), + ({"secret", "other"}, None, ["secret", "other"], ["***", "***"]), + # We don't mask dict _keys_. + ({"secret", "other"}, None, {"data": {"secret": "secret"}}, {"data": {"secret": "***"}}), + ( + # Since this is a sensitive name, all the values should be redacted! + {"secret"}, + "api_key", + {"other": "innoent", "nested": ["x", "y"]}, + {"other": "***", "nested": ["***", "***"]}, + ), + ( + # Test that masking still works based on name even when no patterns given + set(), + 'env', + {'api_key': 'masked based on key name', 'other': 'foo'}, + {'api_key': '***', 'other': 'foo'}, + ), + ], + ) + def test_redact(self, patterns, name, value, expected): + filt = SecretsMasker() + for val in patterns: + filt.add_mask(val) + + assert filt.redact(value, name) == expected + + +class TestShouldHideValueForKey: + @pytest.mark.parametrize( + ("key", "expected_result"), + [ + ('', False), + (None, False), + ("key", False), + ("google_api_key", True), + ("GOOGLE_API_KEY", True), + ("GOOGLE_APIKEY", True), + ], + ) + def test_hiding_defaults(self, key, expected_result): + assert expected_result == should_hide_value_for_key(key) + + @pytest.mark.parametrize( + ("sensitive_variable_fields", "key", "expected_result"), + [ + ('key', 'TRELLO_KEY', True), + ('key', 'TRELLO_API_KEY', True), + ('key', 'GITHUB_APIKEY', True), + ('key, token', 'TRELLO_TOKEN', True), + ('mysecretword, mysensitivekey', 'GITHUB_mysecretword', True), + (None, 'TRELLO_API', False), + ('token', 'TRELLO_KEY', False), + ('token, mysecretword', 'TRELLO_KEY', False), + ], + ) + def test_hiding_config(self, sensitive_variable_fields, key, expected_result): + from airflow.utils.log.secrets_masker import get_sensitive_variables_fields + + with conf_vars({('core', 'sensitive_var_conn_names'): str(sensitive_variable_fields)}): + get_sensitive_variables_fields.cache_clear() + assert expected_result == should_hide_value_for_key(key) + get_sensitive_variables_fields.cache_clear() + + +class ShortExcFormatter(logging.Formatter): + """Don't include full path in exc_info messages""" + + def formatException(self, exc_info): + formatted = super().formatException(exc_info) + return formatted.replace(__file__, ".../" + os.path.basename(__file__)) + + +def lineno(): + """Returns the current line number in our program.""" + return inspect.currentframe().f_back.f_lineno diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py deleted file mode 100644 index 372d93d473676..0000000000000 --- a/tests/utils/test_decorators.py +++ /dev/null @@ -1,74 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest - -import pytest - -from airflow.exceptions import AirflowException -from airflow.utils.decorators import apply_defaults - - -# Essentially similar to airflow.models.BaseOperator -class DummyClass: - @apply_defaults - def __init__(self, test_param, params=None, default_args=None): # pylint: disable=unused-argument - self.test_param = test_param - - -class DummySubClass(DummyClass): - @apply_defaults - def __init__(self, test_sub_param, **kwargs): - super().__init__(**kwargs) - self.test_sub_param = test_sub_param - - -class TestApplyDefault(unittest.TestCase): - def test_apply(self): - dummy = DummyClass(test_param=True) - assert dummy.test_param - - with pytest.raises(AirflowException, match='Argument.*test_param.*required'): - DummySubClass(test_sub_param=True) - - def test_default_args(self): - default_args = {'test_param': True} - dummy_class = DummyClass(default_args=default_args) # pylint: disable=no-value-for-parameter - assert dummy_class.test_param - - default_args = {'test_param': True, 'test_sub_param': True} - dummy_subclass = DummySubClass(default_args=default_args) # pylint: disable=no-value-for-parameter - assert dummy_class.test_param - assert dummy_subclass.test_sub_param - - default_args = {'test_param': True} - dummy_subclass = DummySubClass(default_args=default_args, test_sub_param=True) - assert dummy_class.test_param - assert dummy_subclass.test_sub_param - - with pytest.raises(AirflowException, match='Argument.*test_sub_param.*required'): - DummySubClass(default_args=default_args) # pylint: disable=no-value-for-parameter - - def test_incorrect_default_args(self): - default_args = {'test_param': True, 'extra_param': True} - dummy_class = DummyClass(default_args=default_args) # pylint: disable=no-value-for-parameter - assert dummy_class.test_param - - default_args = {'random_params': True} - with pytest.raises(AirflowException, match='Argument.*test_param.*required'): - DummyClass(default_args=default_args) # pylint: disable=no-value-for-parameter diff --git a/tests/utils/test_process_utils.py b/tests/utils/test_process_utils.py index 21d6cdd98b6e4..eabf94fcf3257 100644 --- a/tests/utils/test_process_utils.py +++ b/tests/utils/test_process_utils.py @@ -90,8 +90,8 @@ def test_reap_process_group(self): assert not psutil.pid_exists(child_pid.value) finally: try: - os.kill(parent_pid.value, signal.SIGKILL) # terminate doesnt work here - os.kill(child_pid.value, signal.SIGKILL) # terminate doesnt work here + os.kill(parent_pid.value, signal.SIGKILL) # terminate doesn't work here + os.kill(child_pid.value, signal.SIGKILL) # terminate doesn't work here except OSError: pass diff --git a/tests/www/api/experimental/test_endpoints.py b/tests/www/api/experimental/test_endpoints.py index bd3dc12858901..8c6734c7a4ca8 100644 --- a/tests/www/api/experimental/test_endpoints.py +++ b/tests/www/api/experimental/test_endpoints.py @@ -55,7 +55,7 @@ def _setup_attrs_base(self, experiemental_api_app, configured_session): def assert_deprecated(self, resp): assert 'true' == resp.headers['Deprecation'] assert re.search( - r'\<.+/upgrading-to-2.html#migration-guide-from-experimental-api-to-stable-api-v1\>; ' + r'<.+/upgrading-to-2.html#migration-guide-from-experimental-api-to-stable-api-v1>; ' 'rel="deprecation"; type="text/html"', resp.headers['Link'], ) diff --git a/tests/www/test_app.py b/tests/www/test_app.py index 47b8f812faf83..2e2579898da7e 100644 --- a/tests/www/test_app.py +++ b/tests/www/test_app.py @@ -223,6 +223,8 @@ def debug_view(): def test_should_set_sqlalchemy_engine_options(self): app = application.cached_app(testing=True) engine_params = {'pool_size': 3, 'pool_recycle': 120, 'pool_pre_ping': True, 'max_overflow': 5} + if app.config['SQLALCHEMY_DATABASE_URI'].startswith('mysql'): + engine_params['isolation_level'] = 'READ COMMITTED' assert app.config['SQLALCHEMY_ENGINE_OPTIONS'] == engine_params @conf_vars( diff --git a/tests/www/test_security.py b/tests/www/test_security.py index 6dabe8cf7313a..73fe4b1f2135e 100644 --- a/tests/www/test_security.py +++ b/tests/www/test_security.py @@ -283,6 +283,7 @@ def test_get_user_roles_for_anonymous_user(self): viewer_role_perms = { (permissions.ACTION_CAN_READ, permissions.RESOURCE_AUDIT_LOG), (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG), + (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_DEPENDENCIES), (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_CODE), (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_RUN), (permissions.ACTION_CAN_READ, permissions.RESOURCE_IMPORT_ERROR), diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index f3f2db0f4ad82..8f381a54b8dfa 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -22,55 +22,12 @@ from urllib.parse import parse_qs from bs4 import BeautifulSoup -from parameterized import parameterized from airflow.www import utils from airflow.www.utils import wrapped_markdown -from tests.test_utils.config import conf_vars class TestUtils(unittest.TestCase): - def test_empty_variable_should_not_be_hidden(self): - assert not utils.should_hide_value_for_key("") - assert not utils.should_hide_value_for_key(None) - - def test_normal_variable_should_not_be_hidden(self): - assert not utils.should_hide_value_for_key("key") - - def test_sensitive_variable_should_be_hidden(self): - assert utils.should_hide_value_for_key("google_api_key") - - def test_sensitive_variable_should_be_hidden_ic(self): - assert utils.should_hide_value_for_key("GOOGLE_API_KEY") - - @parameterized.expand( - [ - ('key', 'TRELLO_KEY', True), - ('key', 'TRELLO_API_KEY', True), - ('key', 'GITHUB_APIKEY', True), - ('key, token', 'TRELLO_TOKEN', True), - ('mysecretword, mysensitivekey', 'GITHUB_mysecretword', True), - ], - ) - def test_sensitive_variable_fields_should_be_hidden( - self, sensitive_variable_fields, key, expected_result - ): - with conf_vars({('admin', 'sensitive_variable_fields'): str(sensitive_variable_fields)}): - assert expected_result == utils.should_hide_value_for_key(key) - - @parameterized.expand( - [ - (None, 'TRELLO_API', False), - ('token', 'TRELLO_KEY', False), - ('token, mysecretword', 'TRELLO_KEY', False), - ], - ) - def test_normal_variable_fields_should_not_be_hidden( - self, sensitive_variable_fields, key, expected_result - ): - with conf_vars({('admin', 'sensitive_variable_fields'): str(sensitive_variable_fields)}): - assert expected_result == utils.should_hide_value_for_key(key) - def check_generate_pages_html(self, current_page, total_pages, window=7, check_middle=False): extra_links = 4 # first, prev, next, last search = "'>\"/>" diff --git a/tests/www/test_views.py b/tests/www/test_views.py index 471f90f4de1b6..db6f5616928bc 100644 --- a/tests/www/test_views.py +++ b/tests/www/test_views.py @@ -17,7 +17,6 @@ # under the License. import copy import html -import io import json import logging.config import os @@ -35,24 +34,16 @@ from urllib.parse import quote_plus import jinja2 -import pytest -from flask import Markup, session as flask_session, template_rendered, url_for +from flask import session as flask_session, template_rendered from parameterized import parameterized -from werkzeug.test import Client -from werkzeug.wrappers import BaseResponse from airflow import models, settings, version from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG -from airflow.configuration import conf, initialize_config +from airflow.configuration import conf from airflow.executors.celery_executor import CeleryExecutor from airflow.jobs.base_job import BaseJob -from airflow.models import DAG, Connection, DagRun, TaskInstance -from airflow.models.baseoperator import BaseOperator, BaseOperatorLink -from airflow.models.renderedtifields import RenderedTaskInstanceFields as RTIF -from airflow.models.serialized_dag import SerializedDagModel -from airflow.operators.bash import BashOperator +from airflow.models import DAG, DagRun, TaskInstance from airflow.operators.dummy import DummyOperator -from airflow.plugins_manager import AirflowPlugin, EntryPointSource from airflow.security import permissions from airflow.ti_deps.dependencies_states import QUEUEABLE_STATES, RUNNABLE_STATES from airflow.utils import dates, timezone @@ -64,13 +55,11 @@ from airflow.www import app as application from airflow.www.extensions import init_views from airflow.www.extensions.init_appbuilder_links import init_appbuilder_links -from airflow.www.views import ConnectionModelView, get_safe_url, truncate_task_duration from tests.test_utils import api_connexion_utils from tests.test_utils.asserts import assert_queries_count from tests.test_utils.config import conf_vars from tests.test_utils.db import clear_db_runs from tests.test_utils.decorators import dont_initialize_flask_app_submodules -from tests.test_utils.mock_plugins import mock_plugin_manager class TemplateWithContext(NamedTuple): @@ -239,281 +228,6 @@ def create_user_and_login(self, username, role_name, perms): self.login(username=username, password=username) -class TestConnectionModelView(TestBase): - def setUp(self): - super().setUp() - - self.connection = { - 'conn_id': 'test_conn', - 'conn_type': 'http', - 'description': 'description', - 'host': 'localhost', - 'port': 8080, - 'username': 'root', - 'password': 'admin', - } - - def tearDown(self): - self.clear_table(Connection) - super().tearDown() - - def test_create_connection(self): - init_views.init_connection_form() - resp = self.client.post('/connection/add', data=self.connection, follow_redirects=True) - self.check_content_in_response('Added Row', resp) - - def test_prefill_form_null_extra(self): - mock_form = mock.Mock() - mock_form.data = {"conn_id": "test", "extra": None} - - cmv = ConnectionModelView() - cmv.prefill_form(form=mock_form, pk=1) - - def test_duplicate_connection(self): - """Test Duplicate multiple connection with suffix""" - conn1 = Connection( - conn_id='test_duplicate_gcp_connection', - conn_type='Google Cloud', - description='Google Cloud Connection', - ) - - conn2 = Connection( - conn_id='test_duplicate_mysql_connection', - conn_type='FTP', - description='MongoDB2', - host='localhost', - schema='airflow', - port=3306, - ) - - conn3 = Connection( - conn_id='test_duplicate_postgres_connection_copy1', - conn_type='FTP', - description='Postgres', - host='localhost', - schema='airflow', - port=3306, - ) - - self.clear_table(Connection) - self.session.add_all([conn1, conn2, conn3]) - self.session.commit() - - mock_form = mock.Mock() - mock_form.data = {"action": "mulduplicate", "rowid": [conn1.id, conn3.id]} - resp = self.client.post('/connection/action_post', data=mock_form.data, follow_redirects=True) - - expected_result = { - 'test_duplicate_gcp_connection', - 'test_duplicate_gcp_connection_copy1', - 'test_duplicate_mysql_connection', - 'test_duplicate_postgres_connection_copy1', - 'test_duplicate_postgres_connection_copy2', - } - response = {conn[0] for conn in self.session.query(Connection.conn_id).all()} - - assert resp.status_code == 200 - assert expected_result == response - - -class TestVariableModelView(TestBase): - def setUp(self): - super().setUp() - self.variable = { - 'key': 'test_key', - 'val': 'text_val', - 'description': 'test_description', - 'is_encrypted': True, - } - - def tearDown(self): - self.clear_table(models.Variable) - super().tearDown() - - def test_can_handle_error_on_decrypt(self): - - # create valid variable - self.client.post('/variable/add', data=self.variable, follow_redirects=True) - - # update the variable with a wrong value, given that is encrypted - Var = models.Variable # pylint: disable=invalid-name - ( - self.session.query(Var) - .filter(Var.key == self.variable['key']) - .update({'val': 'failed_value_not_encrypted'}, synchronize_session=False) - ) - self.session.commit() - - # retrieve Variables page, should not fail and contain the Invalid - # label for the variable - resp = self.client.get('/variable/list', follow_redirects=True) - self.check_content_in_response('Invalid', resp) - - def test_xss_prevention(self): - xss = "/variable/list/" - - resp = self.client.get( - xss, - follow_redirects=True, - ) - assert resp.status_code == 404 - assert "" not in resp.data.decode("utf-8") - - def test_import_variables_no_file(self): - resp = self.client.post('/variable/varimport', follow_redirects=True) - self.check_content_in_response('Missing file or syntax error.', resp) - - def test_import_variables_failed(self): - content = '{"str_key": "str_value"}' - - with mock.patch('airflow.models.Variable.set') as set_mock: - set_mock.side_effect = UnicodeEncodeError - assert self.session.query(models.Variable).count() == 0 - - bytes_content = io.BytesIO(bytes(content, encoding='utf-8')) - - resp = self.client.post( - '/variable/varimport', data={'file': (bytes_content, 'test.json')}, follow_redirects=True - ) - self.check_content_in_response('1 variable(s) failed to be updated.', resp) - - def test_import_variables_success(self): - assert self.session.query(models.Variable).count() == 0 - - content = ( - '{"str_key": "str_value", "int_key": 60, "list_key": [1, 2], "dict_key": {"k_a": 2, "k_b": 3}}' - ) - bytes_content = io.BytesIO(bytes(content, encoding='utf-8')) - - resp = self.client.post( - '/variable/varimport', data={'file': (bytes_content, 'test.json')}, follow_redirects=True - ) - self.check_content_in_response('4 variable(s) successfully updated.', resp) - - def test_description_retrieval(self): - # create valid variable - self.client.post('/variable/add', data=self.variable, follow_redirects=True) - - row = self.session.query(models.Variable.key, models.Variable.description).first() - assert row.key == 'test_key' and row.description == 'test_description' - - -class PluginOperator(BaseOperator): - pass - - -class TestPluginView(TestBase): - def test_should_list_plugins_on_page_with_details(self): - resp = self.client.get('/plugin') - self.check_content_in_response("test_plugin", resp) - self.check_content_in_response("Airflow Plugins", resp) - self.check_content_in_response("source", resp) - self.check_content_in_response("$PLUGINS_FOLDER/test_plugin.py", resp) - - def test_should_list_entrypoint_plugins_on_page_with_details(self): - - mock_plugin = AirflowPlugin() - mock_plugin.name = "test_plugin" - mock_plugin.source = EntryPointSource( - mock.Mock(), mock.Mock(version='1.0.0', metadata={'name': 'test-entrypoint-testpluginview'}) - ) - with mock_plugin_manager(plugins=[mock_plugin]): - resp = self.client.get('/plugin') - - self.check_content_in_response("test_plugin", resp) - self.check_content_in_response("Airflow Plugins", resp) - self.check_content_in_response("source", resp) - self.check_content_in_response("test-entrypoint-testpluginview==1.0.0: ', resp) - - def test_list(self): - self.pool['pool'] = 'test-pool' - self.session.add(models.Pool(**self.pool)) - self.session.commit() - resp = self.client.get('/pool/list/') - # We should see this link - with self.app.test_request_context(): - url = url_for('TaskInstanceModelView.list', _flt_3_pool='test-pool', _flt_3_state='running') - used_tag = Markup("{slots}").format(url=url, slots=0) - - url = url_for('TaskInstanceModelView.list', _flt_3_pool='test-pool', _flt_3_state='queued') - queued_tag = Markup("{slots}").format(url=url, slots=0) - self.check_content_in_response(used_tag, resp) - self.check_content_in_response(queued_tag, resp) - - -class TestMountPoint(unittest.TestCase): - @classmethod - @conf_vars({("webserver", "base_url"): "http://localhost/test"}) - def setUpClass(cls): - application.app = None - application.appbuilder = None - app = application.create_app(testing=True) - app.config['WTF_CSRF_ENABLED'] = False - cls.client = Client(app, BaseResponse) - - @classmethod - def tearDownClass(cls): - application.app = None - application.appbuilder = None - - def test_mount(self): - # Test an endpoint that doesn't need auth! - resp = self.client.get('/test/health') - assert resp.status_code == 200 - assert b"healthy" in resp.data - - def test_not_found(self): - resp = self.client.get('/', follow_redirects=True) - assert resp.status_code == 404 - - def test_index(self): - resp = self.client.get('/test/') - assert resp.status_code == 302 - assert resp.headers['Location'] == 'http://localhost/test/home' - - class TestAirflowBaseViews(TestBase): EXAMPLE_DAG_DEFAULT_DATE = dates.days_ago(2) @@ -523,9 +237,10 @@ def setUpClass(cls): models.DagBag(include_examples=True).sync_to_db() cls.dagbag = models.DagBag(include_examples=True, read_dags_from_db=True) cls.app.dag_bag = cls.dagbag - init_views.init_api_connexion(cls.app) - init_views.init_plugins(cls.app) - init_appbuilder_links(cls.app) + with cls.app.app_context(): + init_views.init_api_connexion(cls.app) + init_views.init_plugins(cls.app) + init_appbuilder_links(cls.app) def setUp(self): super().setUp() @@ -565,7 +280,7 @@ def prepare_dagruns(self): ) def test_index(self): - with assert_queries_count(43): + with assert_queries_count(44): resp = self.client.get('/', follow_redirects=True) self.check_content_in_response('DAGs', resp) @@ -1040,10 +755,10 @@ def test_dag_details(self): resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('DAG Details', resp) - @parameterized.expand(["graph", "tree", "dag_details"]) + @parameterized.expand(["graph", "tree", "calendar", "dag_details"]) def test_view_uses_existing_dagbag(self, endpoint): """ - Test that Graph, Tree & Dag Details View uses the DagBag already created in views.py + Test that Graph, Tree, Calendar & Dag Details View uses the DagBag already created in views.py instead of creating a new one. """ url = f'{endpoint}?dag_id=example_bash_operator' @@ -1112,6 +827,12 @@ def test_graph(self): resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('runme_1', resp) + def test_dag_dependencies(self): + url = 'dag-dependencies' + resp = self.client.get(url, follow_redirects=True) + self.check_content_in_response('child_task1', resp) + self.check_content_in_response('test_trigger_dagrun', resp) + def test_last_dagruns(self): resp = self.client.post('last_dagruns', follow_redirects=True) self.check_content_in_response('example_bash_operator', resp) @@ -1146,6 +867,14 @@ def test_tree_subdag(self): resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('section-1-task-1', resp) + def test_calendar(self): + url = 'calendar?dag_id=example_bash_operator' + resp = self.client.get(url, follow_redirects=True) + dag_run_date = self.bash_dagrun.execution_date.date().isoformat() + expected_data = [{'date': dag_run_date, 'state': State.RUNNING, 'count': 1}] + expected_data_json_escaped = json.dumps(expected_data).replace('"', '\\"').replace(' ', '') + self.check_content_in_response(expected_data_json_escaped, resp) + def test_duration(self): url = 'duration?days=30&dag_id=example_bash_operator' resp = self.client.get(url, follow_redirects=True) @@ -1480,50 +1209,6 @@ def test_page_instance_name_xss_prevention(self): self.check_content_not_in_response(xss_string, resp) -class TestConfigurationView(TestBase): - def setUp(self): - super().setUp() - with mock.patch.dict(os.environ, {"AIRFLOW__CORE__UNIT_TEST_MODE": "False"}): - initialize_config() - - def test_configuration_do_not_expose_config(self): - self.logout() - self.login() - with conf_vars({('webserver', 'expose_config'): 'False'}): - resp = self.client.get('configuration', follow_redirects=True) - self.check_content_in_response( - [ - 'Airflow Configuration', - '# Your Airflow administrator chose not to expose the configuration, ' - 'most likely for security reasons.', - ], - resp, - ) - - def test_configuration_expose_config(self): - self.logout() - self.login() - with conf_vars({('webserver', 'expose_config'): 'True'}): - resp = self.client.get('configuration', follow_redirects=True) - self.check_content_in_response(['Airflow Configuration', 'Running Configuration'], resp) - - -class TestRedocView(TestBase): - @classmethod - def setUpClass(cls): - super().setUpClass() - init_views.init_api_connexion(cls.app) - - def test_should_render_template(self): - with self.capture_templates() as templates: - resp = self.client.get('redoc') - self.check_content_in_response('Redoc', resp) - - assert len(templates) == 1 - assert templates[0].name == 'airflow/redoc.html' - assert templates[0].local_context == {'openapi_spec_url': '/api/v1/openapi.yaml'} - - class TestLogView(TestBase): DAG_ID = 'dag_for_testing_log_view' DAG_ID_REMOVED = 'removed_dag_for_testing_log_view' @@ -1570,6 +1255,8 @@ def setUp(self): self.login() dagbag = self.app.dag_bag + dagbag.dags.pop(self.DAG_ID, None) + dagbag.dags.pop(self.DAG_ID_REMOVED, None) dag = DAG(self.DAG_ID, start_date=self.DEFAULT_DATE) dag_removed = DAG(self.DAG_ID_REMOVED, start_date=self.DEFAULT_DATE) dagbag.bag_dag(dag=dag, root_dag=dag) @@ -1844,6 +1531,7 @@ def __init__(self, test, endpoint): def setup(self): dagbag = self.test.app.dag_bag + dagbag.dags.pop(self.DAG_ID, None) dag = DAG(self.DAG_ID, start_date=self.DEFAULT_DATE) dagbag.bag_dag(dag=dag, root_dag=dag) for run_data in self.RUNS_DATA: @@ -2942,751 +2630,3 @@ def test_refresh_failure_for_viewer(self): self.login(username='test_viewer', password='test_viewer') resp = self.client.post('refresh?dag_id=example_bash_operator') self.check_content_in_response('Redirecting', resp, resp_code=302) - - -class TestTaskInstanceView(TestBase): - TI_ENDPOINT = '/taskinstance/list/?_flt_0_execution_date={}' - - def test_start_date_filter(self): - resp = self.client.get(self.TI_ENDPOINT.format(self.percent_encode('2018-10-09 22:44:31'))) - # We aren't checking the logic of the date filter itself (that is built - # in to FAB) but simply that our UTC conversion was run - i.e. it - # doesn't blow up! - self.check_content_in_response('List Task Instance', resp) - - -class TestTaskRescheduleView(TestBase): - TI_ENDPOINT = '/taskreschedule/list/?_flt_0_execution_date={}' - - def test_start_date_filter(self): - resp = self.client.get(self.TI_ENDPOINT.format(self.percent_encode('2018-10-09 22:44:31'))) - # We aren't checking the logic of the date filter itself (that is built - # in to FAB) but simply that our UTC conversion was run - i.e. it - # doesn't blow up! - self.check_content_in_response('List Task Reschedule', resp) - - -class TestRenderedView(TestBase): - def setUp(self): - - self.default_date = datetime(2020, 3, 1) - self.dag = DAG( - "testdag", - start_date=self.default_date, - user_defined_filters={"hello": lambda name: f'Hello {name}'}, - user_defined_macros={"fullname": lambda fname, lname: f'{fname} {lname}'}, - ) - self.task1 = BashOperator(task_id='task1', bash_command='{{ task_instance_key_str }}', dag=self.dag) - self.task2 = BashOperator( - task_id='task2', bash_command='echo {{ fullname("Apache", "Airflow") | hello }}', dag=self.dag - ) - SerializedDagModel.write_dag(self.dag) - with create_session() as session: - session.query(RTIF).delete() - - self.app.dag_bag = mock.MagicMock(**{'get_dag.return_value': self.dag}) - super().setUp() - - def tearDown(self) -> None: - super().tearDown() - with create_session() as session: - session.query(RTIF).delete() - - def test_rendered_template_view(self): - """ - Test that the Rendered View contains the values from RenderedTaskInstanceFields - """ - assert self.task1.bash_command == '{{ task_instance_key_str }}' - ti = TaskInstance(self.task1, self.default_date) - - with create_session() as session: - session.add(RTIF(ti)) - - url = 'rendered-templates?task_id=task1&dag_id=testdag&execution_date={}'.format( - self.percent_encode(self.default_date) - ) - - resp = self.client.get(url, follow_redirects=True) - self.check_content_in_response("testdag__task1__20200301", resp) - - def test_rendered_template_view_for_unexecuted_tis(self): - """ - Test that the Rendered View is able to show rendered values - even for TIs that have not yet executed - """ - assert self.task1.bash_command == '{{ task_instance_key_str }}' - - url = 'rendered-templates?task_id=task1&dag_id=task1&execution_date={}'.format( - self.percent_encode(self.default_date) - ) - - resp = self.client.get(url, follow_redirects=True) - self.check_content_in_response("testdag__task1__20200301", resp) - - def test_user_defined_filter_and_macros_raise_error(self): - """ - Test that the Rendered View is able to show rendered values - even for TIs that have not yet executed - """ - self.app.dag_bag = mock.MagicMock( - **{'get_dag.return_value': SerializedDagModel.get(self.dag.dag_id).dag} - ) - assert self.task2.bash_command == 'echo {{ fullname("Apache", "Airflow") | hello }}' - - url = 'rendered-templates?task_id=task2&dag_id=testdag&execution_date={}'.format( - self.percent_encode(self.default_date) - ) - - resp = self.client.get(url, follow_redirects=True) - self.check_content_not_in_response("echo Hello Apache Airflow", resp) - self.check_content_in_response( - "Webserver does not have access to User-defined Macros or Filters " - "when Dag Serialization is enabled. Hence for the task that have not yet " - "started running, please use 'airflow tasks render' for debugging the " - "rendering of template_fields.

    OriginalError: no filter named 'hello'", - resp, - ) - - -class TestTriggerDag(TestBase): - def setUp(self): - super().setUp() - models.DagBag().get_dag("example_bash_operator").sync_to_db(session=self.session) - self.session.commit() - - def test_trigger_dag_button_normal_exist(self): - resp = self.client.get('/', follow_redirects=True) - assert '/trigger?dag_id=example_bash_operator' in resp.data.decode('utf-8') - assert "return confirmDeleteDag(this, 'example_bash_operator')" in resp.data.decode('utf-8') - - @pytest.mark.quarantined - def test_trigger_dag_button(self): - - test_dag_id = "example_bash_operator" - - DR = models.DagRun # pylint: disable=invalid-name - self.session.query(DR).delete() - self.session.commit() - - self.client.post(f'trigger?dag_id={test_dag_id}') - - run = self.session.query(DR).filter(DR.dag_id == test_dag_id).first() - assert run is not None - assert DagRunType.MANUAL in run.run_id - assert run.run_type == DagRunType.MANUAL - - @pytest.mark.quarantined - def test_trigger_dag_conf(self): - - test_dag_id = "example_bash_operator" - conf_dict = {'string': 'Hello, World!'} - - DR = models.DagRun # pylint: disable=invalid-name - self.session.query(DR).delete() - self.session.commit() - - self.client.post(f'trigger?dag_id={test_dag_id}', data={'conf': json.dumps(conf_dict)}) - - run = self.session.query(DR).filter(DR.dag_id == test_dag_id).first() - assert run is not None - assert DagRunType.MANUAL in run.run_id - assert run.run_type == DagRunType.MANUAL - assert run.conf == conf_dict - - def test_trigger_dag_conf_malformed(self): - test_dag_id = "example_bash_operator" - - DR = models.DagRun # pylint: disable=invalid-name - self.session.query(DR).delete() - self.session.commit() - - response = self.client.post(f'trigger?dag_id={test_dag_id}', data={'conf': '{"a": "b"'}) - self.check_content_in_response('Invalid JSON configuration', response) - - run = self.session.query(DR).filter(DR.dag_id == test_dag_id).first() - assert run is None - - def test_trigger_dag_form(self): - test_dag_id = "example_bash_operator" - resp = self.client.get(f'trigger?dag_id={test_dag_id}') - self.check_content_in_response(f'Trigger DAG: {test_dag_id}', resp) - - @parameterized.expand( - [ - ("javascript:alert(1)", "/home"), - ("http://google.com", "/home"), - ("36539'%3balert(1)%2f%2f166", "/home"), - ( - "%2Ftree%3Fdag_id%3Dexample_bash_operator';alert(33)//", - "/home", - ), - ("%2Ftree%3Fdag_id%3Dexample_bash_operator", "/tree?dag_id=example_bash_operator"), - ("%2Fgraph%3Fdag_id%3Dexample_bash_operator", "/graph?dag_id=example_bash_operator"), - ] - ) - def test_trigger_dag_form_origin_url(self, test_origin, expected_origin): - test_dag_id = "example_bash_operator" - - resp = self.client.get(f'trigger?dag_id={test_dag_id}&origin={test_origin}') - self.check_content_in_response( - '