From cff724e0d69518e95d0378f2ab5b85c8138a4f4e Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 23 Jan 2025 00:18:52 +0100 Subject: [PATCH] Move standad provider to the new structure This one also adds some missing steps in provider migration script. improves the output and instructions and make it more robust for various variations of provider features and usage of imports. With those changes `standard` provider was 100% moved automatically. --- .github/boring-cyborg.yml | 13 +- .pre-commit-config.yaml | 15 +- .../templates/pyproject_TEMPLATE.toml.jinja2 | 4 +- dev/breeze/tests/test_selective_checks.py | 6 +- dev/moving_providers/README.md | 41 ++- dev/moving_providers/move_providers.py | 311 ++++++++++++++---- docs/.gitignore | 2 + .../changelog.rst | 25 -- .../changelog.rst | 25 -- providers/airbyte/pyproject.toml | 7 +- .../airbyte/tests/provider_tests/__init__.py | 2 - providers/alibaba/README.rst | 64 ++++ .../docs}/.latest-doc-only-change.txt | 0 .../docs/changelog.rst} | 0 .../alibaba/docs}/commits.rst | 0 .../alibaba/docs}/connections/alibaba.rst | 0 .../alibaba/docs}/index.rst | 0 .../installing-providers-from-sources.rst | 0 .../alibaba/docs}/logging/index.rst | 0 .../docs}/logging/oss-task-handler.rst | 0 .../docs}/operators/analyticdb_spark.rst | 2 +- .../alibaba/docs}/operators/index.rst | 0 .../alibaba/docs}/operators/oss.rst | 2 +- .../alibaba/docs}/redirects.txt | 0 .../alibaba/docs}/security.rst | 0 .../providers => }/alibaba/provider.yaml | 6 - providers/alibaba/pyproject.toml | 80 +++++ .../src/airflow/providers/alibaba/LICENSE | 201 +++++++++++ .../src/airflow/providers/alibaba/__init__.py | 0 .../providers/alibaba/cloud/__init__.py | 0 .../providers/alibaba/cloud/hooks/__init__.py | 0 .../alibaba/cloud/hooks/analyticdb_spark.py | 0 .../providers/alibaba/cloud/hooks/oss.py | 0 .../providers/alibaba/cloud/log/__init__.py | 0 .../alibaba/cloud/log/oss_task_handler.py | 0 .../alibaba/cloud/operators/__init__.py | 0 .../cloud/operators/analyticdb_spark.py | 0 .../providers/alibaba/cloud/operators/oss.py | 0 .../alibaba/cloud/sensors/__init__.py | 0 .../alibaba/cloud/sensors/analyticdb_spark.py | 0 .../alibaba/cloud/sensors/oss_key.py | 0 .../providers/alibaba/get_provider_info.py | 120 +++++++ providers/alibaba/tests/conftest.py | 32 ++ .../alibaba/tests/provider_tests/__init__.py | 17 + .../tests/provider_tests/alibaba}/__init__.py | 0 .../provider_tests/alibaba/cloud}/__init__.py | 0 .../alibaba/cloud/hooks}/__init__.py | 0 .../cloud/hooks/test_analyticdb_spark.py | 3 +- .../alibaba/cloud/hooks/test_oss.py | 3 +- .../alibaba/cloud/log}/__init__.py | 0 .../cloud/log/test_oss_task_handler.py | 0 .../alibaba/cloud/operators}/__init__.py | 0 .../cloud/operators/test_analyticdb_spark.py | 0 .../alibaba/cloud/operators/test_oss.py | 0 .../alibaba/cloud/sensors}/__init__.py | 0 .../cloud/sensors/test_analyticdb_spark.py | 0 .../alibaba/cloud/sensors/test_oss_key.py | 0 .../alibaba/cloud/utils}/__init__.py | 0 .../cloud/utils/analyticdb_spark_mock.py | 0 .../alibaba/cloud/utils/oss_mock.py | 0 .../alibaba/cloud/utils/test_utils.py | 0 .../tests/system/alibaba}/__init__.py | 0 .../system/alibaba/example_adb_spark_batch.py | 0 .../system/alibaba/example_adb_spark_sql.py | 0 .../system/alibaba/example_oss_bucket.py | 0 .../system/alibaba/example_oss_object.py | 0 providers/apache/iceberg/pyproject.toml | 7 +- .../iceberg/tests/provider_tests/__init__.py | 2 - .../tests/provider_tests/apache/__init__.py | 2 - providers/celery/pyproject.toml | 7 +- .../celery/tests/provider_tests/__init__.py | 2 - providers/edge/pyproject.toml | 7 +- .../edge/tests/provider_tests/__init__.py | 2 - providers/standard/README.rst | 62 ++++ .../docs/changelog.rst} | 0 .../standard/docs}/commits.rst | 0 .../standard/docs}/configurations-ref.rst | 0 .../standard/docs}/index.rst | 0 .../installing-providers-from-sources.rst | 0 .../standard/docs}/operators/bash.rst | 0 .../standard/docs}/operators/datetime.rst | 0 .../standard/docs}/operators/index.rst | 0 .../standard/docs}/operators/python.rst | 0 .../standard/docs}/security.rst | 0 .../standard/docs}/sensors/bash.rst | 0 .../standard/docs}/sensors/datetime.rst | 0 .../docs}/sensors/external_task_sensor.rst | 0 .../standard/docs}/sensors/file.rst | 0 .../standard/docs}/sensors/index.rst | 0 .../standard/docs}/sensors/python.rst | 0 .../providers => }/standard/provider.yaml | 4 - providers/standard/pyproject.toml | 78 +++++ .../src/airflow/providers/standard/LICENSE | 201 +++++++++++ .../airflow/providers/standard/__init__.py | 0 .../providers/standard/get_provider_info.py | 108 ++++++ .../providers/standard/hooks}/__init__.py | 0 .../providers/standard/hooks/filesystem.py | 0 .../providers/standard/hooks/package_index.py | 0 .../providers/standard/hooks/subprocess.py | 0 .../providers/standard}/operators/__init__.py | 0 .../providers/standard/operators/bash.py | 0 .../providers/standard/operators/datetime.py | 0 .../standard/operators/generic_transfer.py | 0 .../standard/operators/latest_only.py | 0 .../providers/standard/operators/python.py | 0 .../standard/operators/trigger_dagrun.py | 0 .../providers/standard/operators/weekday.py | 0 .../providers/standard}/sensors/__init__.py | 0 .../providers/standard/sensors/bash.py | 0 .../providers/standard/sensors/date_time.py | 0 .../standard/sensors/external_task.py | 0 .../providers/standard/sensors/filesystem.py | 0 .../providers/standard/sensors/python.py | 0 .../providers/standard/sensors/time.py | 0 .../providers/standard/sensors/time_delta.py | 0 .../providers/standard/sensors/weekday.py | 0 .../providers/standard/triggers}/__init__.py | 0 .../standard/triggers/external_task.py | 0 .../providers/standard/triggers/file.py | 0 .../providers/standard/triggers/temporal.py | 0 .../providers/standard/utils}/__init__.py | 0 .../standard/utils/python_virtualenv.py | 0 .../utils/python_virtualenv_script.jinja2 | 0 .../providers/standard/utils/sensor_helper.py | 0 .../providers/standard/version_compat.py | 0 providers/standard/tests/conftest.py | 32 ++ .../standard/tests/provider_tests/__init__.py | 17 + .../provider_tests/standard}/__init__.py | 0 .../standard/decorators}/__init__.py | 0 .../standard}/decorators/test_python.py | 4 +- .../standard/hooks}/__init__.py | 0 .../standard/hooks/test_filesystem.py | 0 .../standard/hooks/test_package_index.py | 0 .../standard/hooks/test_subprocess.py | 0 .../standard/operators}/__init__.py | 0 .../standard/operators/test_bash.py | 0 .../standard/operators/test_datetime.py | 0 .../operators/test_generic_transfer.py | 0 .../operators/test_latest_only_operator.py | 0 .../standard/operators/test_python.py | 0 .../standard/operators/test_weekday.py | 0 .../standard/sensors}/__init__.py | 0 .../standard/sensors/test_bash.py | 0 .../standard/sensors/test_date_time.py | 0 .../standard/sensors/test_python.py | 3 +- .../standard/sensors/test_time.py | 0 .../standard/sensors/test_time_delta.py | 0 .../standard/sensors/test_weekday.py | 0 .../standard/triggers}/__init__.py | 0 .../standard/triggers/test_external_task.py | 0 .../standard/triggers/test_file.py | 0 .../standard/triggers/test_temporal.py | 0 .../provider_tests/standard/utils/__init__.py | 16 + .../standard/utils/test_python_virtualenv.py | 0 pyproject.toml | 22 +- scripts/in_container/run_fix_ownership.py | 4 +- tests_common/test_utils/__init__.py | 2 +- 157 files changed, 1382 insertions(+), 181 deletions(-) delete mode 100644 docs/apache-airflow-providers-alibaba/changelog.rst delete mode 100644 docs/apache-airflow-providers-standard/changelog.rst create mode 100644 providers/alibaba/README.rst rename providers/{src/airflow/providers/alibaba => alibaba/docs}/.latest-doc-only-change.txt (100%) rename providers/{src/airflow/providers/alibaba/CHANGELOG.rst => alibaba/docs/changelog.rst} (100%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/commits.rst (100%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/connections/alibaba.rst (100%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/index.rst (100%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/installing-providers-from-sources.rst (100%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/logging/index.rst (100%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/logging/oss-task-handler.rst (100%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/operators/analyticdb_spark.rst (94%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/operators/index.rst (100%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/operators/oss.rst (95%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/redirects.txt (100%) rename {docs/apache-airflow-providers-alibaba => providers/alibaba/docs}/security.rst (100%) rename providers/{src/airflow/providers => }/alibaba/provider.yaml (95%) create mode 100644 providers/alibaba/pyproject.toml create mode 100644 providers/alibaba/src/airflow/providers/alibaba/LICENSE rename providers/{ => alibaba}/src/airflow/providers/alibaba/__init__.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/__init__.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/hooks/__init__.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/hooks/oss.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/log/__init__.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/log/oss_task_handler.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/operators/__init__.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/operators/oss.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/sensors/__init__.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py (100%) rename providers/{ => alibaba}/src/airflow/providers/alibaba/cloud/sensors/oss_key.py (100%) create mode 100644 providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py create mode 100644 providers/alibaba/tests/conftest.py create mode 100644 providers/alibaba/tests/provider_tests/__init__.py rename providers/{src/airflow/providers/standard/hooks => alibaba/tests/provider_tests/alibaba}/__init__.py (100%) rename providers/{src/airflow/providers/standard/operators => alibaba/tests/provider_tests/alibaba/cloud}/__init__.py (100%) rename providers/{src/airflow/providers/standard/sensors => alibaba/tests/provider_tests/alibaba/cloud/hooks}/__init__.py (100%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/hooks/test_analyticdb_spark.py (98%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/hooks/test_oss.py (98%) rename providers/{src/airflow/providers/standard/triggers => alibaba/tests/provider_tests/alibaba/cloud/log}/__init__.py (100%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/log/test_oss_task_handler.py (100%) rename providers/{src/airflow/providers/standard/utils => alibaba/tests/provider_tests/alibaba/cloud/operators}/__init__.py (100%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/operators/test_analyticdb_spark.py (100%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/operators/test_oss.py (100%) rename providers/{tests/alibaba => alibaba/tests/provider_tests/alibaba/cloud/sensors}/__init__.py (100%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/sensors/test_analyticdb_spark.py (100%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/sensors/test_oss_key.py (100%) rename providers/{tests/alibaba/cloud => alibaba/tests/provider_tests/alibaba/cloud/utils}/__init__.py (100%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/utils/analyticdb_spark_mock.py (100%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/utils/oss_mock.py (100%) rename providers/{tests => alibaba/tests/provider_tests}/alibaba/cloud/utils/test_utils.py (100%) rename providers/{tests/alibaba/cloud/hooks => alibaba/tests/system/alibaba}/__init__.py (100%) rename providers/{ => alibaba}/tests/system/alibaba/example_adb_spark_batch.py (100%) rename providers/{ => alibaba}/tests/system/alibaba/example_adb_spark_sql.py (100%) rename providers/{ => alibaba}/tests/system/alibaba/example_oss_bucket.py (100%) rename providers/{ => alibaba}/tests/system/alibaba/example_oss_object.py (100%) create mode 100644 providers/standard/README.rst rename providers/{src/airflow/providers/standard/CHANGELOG.rst => standard/docs/changelog.rst} (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/commits.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/configurations-ref.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/index.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/installing-providers-from-sources.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/operators/bash.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/operators/datetime.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/operators/index.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/operators/python.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/security.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/sensors/bash.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/sensors/datetime.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/sensors/external_task_sensor.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/sensors/file.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/sensors/index.rst (100%) rename {docs/apache-airflow-providers-standard => providers/standard/docs}/sensors/python.rst (100%) rename providers/{src/airflow/providers => }/standard/provider.yaml (97%) create mode 100644 providers/standard/pyproject.toml create mode 100644 providers/standard/src/airflow/providers/standard/LICENSE rename providers/{ => standard}/src/airflow/providers/standard/__init__.py (100%) create mode 100644 providers/standard/src/airflow/providers/standard/get_provider_info.py rename providers/{tests/alibaba/cloud/log => standard/src/airflow/providers/standard/hooks}/__init__.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/hooks/filesystem.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/hooks/package_index.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/hooks/subprocess.py (100%) rename providers/{tests/alibaba/cloud => standard/src/airflow/providers/standard}/operators/__init__.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/operators/bash.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/operators/datetime.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/operators/generic_transfer.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/operators/latest_only.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/operators/python.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/operators/trigger_dagrun.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/operators/weekday.py (100%) rename providers/{tests/alibaba/cloud => standard/src/airflow/providers/standard}/sensors/__init__.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/sensors/bash.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/sensors/date_time.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/sensors/external_task.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/sensors/filesystem.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/sensors/python.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/sensors/time.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/sensors/time_delta.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/sensors/weekday.py (100%) rename providers/{tests/alibaba/cloud/utils => standard/src/airflow/providers/standard/triggers}/__init__.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/triggers/external_task.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/triggers/file.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/triggers/temporal.py (100%) rename providers/{tests/standard => standard/src/airflow/providers/standard/utils}/__init__.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/utils/python_virtualenv.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/utils/python_virtualenv_script.jinja2 (100%) rename providers/{ => standard}/src/airflow/providers/standard/utils/sensor_helper.py (100%) rename providers/{ => standard}/src/airflow/providers/standard/version_compat.py (100%) create mode 100644 providers/standard/tests/conftest.py create mode 100644 providers/standard/tests/provider_tests/__init__.py rename providers/{tests/standard/hooks => standard/tests/provider_tests/standard}/__init__.py (100%) rename providers/{tests/standard/operators => standard/tests/provider_tests/standard/decorators}/__init__.py (100%) rename {tests => providers/standard/tests/provider_tests/standard}/decorators/test_python.py (99%) rename providers/{tests/standard/sensors => standard/tests/provider_tests/standard/hooks}/__init__.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/hooks/test_filesystem.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/hooks/test_package_index.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/hooks/test_subprocess.py (100%) rename providers/{tests/standard/triggers => standard/tests/provider_tests/standard/operators}/__init__.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/operators/test_bash.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/operators/test_datetime.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/operators/test_generic_transfer.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/operators/test_latest_only_operator.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/operators/test_python.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/operators/test_weekday.py (100%) rename providers/{tests/standard/utils => standard/tests/provider_tests/standard/sensors}/__init__.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/sensors/test_bash.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/sensors/test_date_time.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/sensors/test_python.py (97%) rename providers/{tests => standard/tests/provider_tests}/standard/sensors/test_time.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/sensors/test_time_delta.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/sensors/test_weekday.py (100%) rename providers/{tests/system/alibaba => standard/tests/provider_tests/standard/triggers}/__init__.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/triggers/test_external_task.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/triggers/test_file.py (100%) rename providers/{tests => standard/tests/provider_tests}/standard/triggers/test_temporal.py (100%) create mode 100644 providers/standard/tests/provider_tests/standard/utils/__init__.py rename providers/{tests => standard/tests/provider_tests}/standard/utils/test_python_virtualenv.py (100%) diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 2c97219a4e5bd5..c2dd99bba8f840 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -22,10 +22,7 @@ labelPRBasedOnFilePath: - providers/airbyte/** provider:alibaba: - - providers/src/airflow/providers/alibaba/**/* - - docs/apache-airflow-providers-alibaba/**/* - - providers/tests/alibaba/**/* - - providers/tests/system/alibaba/**/* + - providers/alibaba/** provider:amazon: - providers/src/airflow/providers/amazon/**/* @@ -179,9 +176,7 @@ labelPRBasedOnFilePath: - providers/tests/system/common/sql/**/* provider:standard: - - providers/src/airflow/providers/standard/**/* - - docs/apache-airflow-providers-standard/**/* - - providers/tests/standard/**/* + - providers/standard/** provider:databricks: - providers/src/airflow/providers/databricks/**/* @@ -647,12 +642,12 @@ labelPRBasedOnFilePath: - airflow/cli/commands/local_commands/triggerer_command.py - airflow/jobs/triggerer_job_runner.py - airflow/models/trigger.py - - providers/src/airflow/providers/standard/triggers/**/* + - providers/standard/src/airflow/providers/standard/triggers/**/* - tests/cli/commands/local_commands/test_triggerer_command.py - tests/jobs/test_triggerer_job.py - tests/models/test_trigger.py - tests/jobs/test_triggerer_job_logging.py - - providers/tests/standard/triggers/**/* + - providers/standard/tests/provider_tests/standard/triggers/**/* area:Serialization: - airflow/serialization/**/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8431755955c122..e1165c455365b2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -724,6 +724,8 @@ repos: ^airflow/operators/.*$| ^providers/src/airflow/providers/.*$| ^providers/src/airflow/providers/standard/sensors/.*$| + ^providers/.*/src/airflow/providers/.*$| + ^providers/.*/src/airflow/providers/standard/sensors/.*$| ^dev/provider_packages/.*$ - id: check-base-operator-usage language: pygrep @@ -733,8 +735,9 @@ repos: pass_filenames: true files: > (?x) - ^providers/src/airflow/providers/.*\.py$ - exclude: providers/src/airflow/providers/standard/operators/bash.py|providers/src/airflow/providers/standard/operators/python.py|providers/src/airflow/providers/standard/sensors/external_task.py + ^providers/src/airflow/providers/.*\.py$| + ^providers/.*/src/airflow/providers/.*\.py$ + exclude: providers/standard/.*/.*\.py$ - id: check-get-lineage-collector-providers language: python name: Check providers import hook lineage code from compat @@ -1206,9 +1209,11 @@ repos: ^airflow/utils/helpers.py$ | ^airflow/utils/log/secrets_masker.py$ | ^providers/src/airflow/providers/ | - ^(providers/)?tests/ | - task_sdk/src/airflow/sdk/definitions/dag.py$ | - task_sdk/src/airflow/sdk/definitions/_internal/node.py$ | + ^tests/ | + ^providers/tests/ | + ^providers/.*/tests/ | + ^task_sdk/src/airflow/sdk/definitions/dag.py$ | + ^task_sdk/src/airflow/sdk/definitions/_internal/node.py$ | ^dev/.*\.py$ | ^scripts/.*\.py$ | ^docker_tests/.*$ | diff --git a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2 b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2 index 330b0c9f19e9e0..5da149fa0d542d 100644 --- a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2 +++ b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2 @@ -63,9 +63,9 @@ classifiers = [ "Framework :: Apache Airflow", "Framework :: Apache Airflow :: Provider", "License :: OSI Approved :: Apache Software License", - {%- for python_version in SUPPORTED_PYTHON_VERSIONS %} + {% for python_version in SUPPORTED_PYTHON_VERSIONS %} "Programming Language :: Python :: {{ python_version }}", - {%- endfor %} + {% endfor %} "Topic :: System :: Monitoring", ] requires-python = "~=3.9" diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 899f987db5b065..cc2350e823ab13 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -253,7 +253,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ), ( pytest.param( - ("providers/src/airflow/providers/standard/operators/python.py",), + ("providers/standard/src/airflow/providers/standard/operators/python.py",), { "selected-providers-list-as-string": None, "all-python-versions": "['3.9']", @@ -823,7 +823,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Only Always and common providers tests should run when only common.io and tests/always changed", ), pytest.param( - ("providers/src/airflow/providers/standard/operators/bash.py",), + ("providers/standard/src/airflow/providers/standard/operators/bash.py",), { "selected-providers-list-as-string": "common.compat standard", "all-python-versions": "['3.9']", @@ -848,7 +848,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Providers standard tests and Serialization tests to run when airflow bash.py changed", ), pytest.param( - ("providers/src/airflow/providers/standard/operators/bash.py",), + ("providers/standard/src/airflow/providers/standard/operators/bash.py",), { "selected-providers-list-as-string": None, "all-python-versions": "['3.9']", diff --git a/dev/moving_providers/README.md b/dev/moving_providers/README.md index abd885b69f38a8..4edcb422d848b2 100644 --- a/dev/moving_providers/README.md +++ b/dev/moving_providers/README.md @@ -24,6 +24,8 @@ - [Moving providers to new structure](#moving-providers-to-new-structure) - [How to use the script](#how-to-use-the-script) - [Options](#options) +- [What happens under the hood](#what-happens-under-the-hood) +- [What to do next](#what-to-do-next) @@ -44,12 +46,21 @@ some manual adjustments needed in more complex cases. The script follows https://peps.python.org/pep-0723/ and uses inlined dependencies - so it can be run as-is by modern tools without creating dedicated virtualenv - the virtualenv with dependencies is -created on-the-fly by PEP 723 compatible tools. For example one can use uv to run it: +created on-the-fly by PEP 723 compatible tools. + +For example this one will make a dry-run of moving Alibaba provider: ```shell uv run dev/moving_providers/move_providers.py alibaba ``` +And this one will perform update and move the Alibaba provider: + + +```shell +uv run dev/moving_providers/move_providers.py alibaba --perform-update +``` + ## Options @@ -60,9 +71,8 @@ uv run dev/moving_providers/move_providers.py alibaba > uv run dev/moving_providers/move_providers.py --help > ``` -By default the script runs in `--dry-run` mode, which means it will not make any changes to the file system, -but will print what it would do. To actually move the files, you need to pass `--no-dry-run` option and you -will be asked to commit the code and create a PR: +By default the script runs in dry run mode, which means it will not make any changes to the file system, +but will print what it would do. To actually move the files, you need to pass `--perform-update` flag. ```shell uv run dev/moving_providers/move_providers.py alibaba --no-dry-run @@ -85,3 +95,26 @@ You can also specify `--quiet` option to see less output: ```shell uv run dev/moving_providers/move_providers.py alibaba --quiet ``` + +# What happens under the hood + +When you run the script with `--perform-update` flag, you will see the diff of the changes +that the script made, and you will be able to scroll through it (with your configured editor) +to verify that it looks good. + +The script will: + +* move the provider to the new structure and apply fixes +* build CI image to add the new provider in the image packages +* run static checks to verify that the moved provider code is good and apply auto-fixes in some cases + +# What to do next + +After all that you need to fix all potential static check problems, run all the tests for the provider and +fix any issues that might happen: + +1) Fix all the static check errors, add them to git +2) run `breeze testing providers-tests --test-type 'Providers[LIST_OF_PROVIDER_IDS_MOVED]'` and fix all tests. +3) Add changes to git, create branch, commit the changes and create a PR! + +Good luck! diff --git a/dev/moving_providers/move_providers.py b/dev/moving_providers/move_providers.py index 2a4316e2a549f9..012cf8bcb7152c 100755 --- a/dev/moving_providers/move_providers.py +++ b/dev/moving_providers/move_providers.py @@ -102,9 +102,8 @@ def _do_stuff( console.print(Syntax("\n".join(diff), "diff", theme="ansi_dark")) console.print() elif updated_content and not from_content and to_path: + console.print(f"\n[yellow]Creating {to_path}:\n") console.print(Syntax(updated_str, syntax, theme="ansi_dark")) - elif updated_content and to_path: - console.print(f"\n[yellow]Creating {to_path}\n") elif not from_content and not updated_content and from_path and to_path and delete_from: console.print(f"\n[yellow]Moving[/] {from_path} -> {to_path}\n") elif not from_content and not updated_content and from_path and to_path and not delete_from: @@ -138,13 +137,25 @@ def _do_stuff( console.print(f"\n[yellow]Deleted {from_path}\n") +def _replace_string(path: Path, old: str, new: str): + content = path.read_text() + count_occurrences = content.count(old) + if count_occurrences: + new_content = content.replace(old, new) + console.print( + f"\n[bright_blue]Replacing `{old}` with `{new}` in `{path}`: " + f"{count_occurrences} occurrences found\n" + ) + if not is_dry_run: + path.write_text(new_content) + + @click.command() @click.argument("provider_ids", type=click.Choice(_get_all_old_providers()), required=True, nargs=-1) @click.option( - "--dry-run/--no-dry-run", - default=True, - help="Whether to run the command without making changes.", - show_default=True, + "--perform-update", + help="By default the command performs dry-run, explaining what will happen. With `--perform-update` " + "it will actually do the job.", is_flag=True, ) @click.option( @@ -163,17 +174,24 @@ def _do_stuff( is_flag=True, ) def move_providers( - provider_ids: tuple[str, ...], dry_run: bool, skip_build_file_generation: bool, verbose: bool, quiet: bool + provider_ids: tuple[str, ...], + perform_update: bool, + skip_build_file_generation: bool, + verbose: bool, + quiet: bool, ): if quiet and verbose: console.print("\n[red]Cannot use --quiet and --verbose at the same time\n") sys.exit(1) - if dry_run: - console.print("\n[yellow]Running in dry-run mode, no changes will be made\n") global is_quiet, is_verbose, is_dry_run is_quiet = quiet is_verbose = verbose - is_dry_run = dry_run + is_dry_run = not perform_update + if is_dry_run: + console.print( + "\n[yellow]Running in dry-run mode, no changes will be made.[/]\n\n" + "Add `--perform-update` flag to actually make the change.\n" + ) console.print("\n[blue]Moving providers:[/]\n") console.print("* " + "\n *".join(provider_ids)) @@ -186,22 +204,28 @@ def move_providers( console.print() count_providers = len(_get_all_old_providers()) - if not dry_run: + if perform_update: subprocess.run("git add .", shell=True, check=True) if not skip_build_file_generation: subprocess.run("pre-commit run update-providers-build-files", shell=True, check=False) - subprocess.run("breeze ci-image build --python 3.9 --answer yes", shell=True, check=True) - subprocess.run("git add . ", shell=True, check=False) + subprocess.run("git add . ", shell=True, check=True) subprocess.run("git diff HEAD", shell=True, check=False) + subprocess.run("uv sync --all-extras", shell=True, check=False) + subprocess.run("breeze static-checks --force-build", shell=True, check=False) + subprocess.run("git add . ", shell=True, check=True) console.print("\n[bright_green]First part of migration is complete[/].\n") - console.print("[yellow]Next steps:") - console.print("* run `pre-commit run`") - console.print("* fix all remaining errors, ") - console.print("* create branch, commit the changes and create a PR!\n") console.print( - f"\nAfter the PR is merged there will be {count_providers - len(provider_ids)} providers " + f"\nAfter you create PR and it will be merged there will be {count_providers - len(provider_ids)} providers " f"left in the old location.\n" ) + console.print("[yellow]Next steps:[/]\n") + console.print(" 1) Fix all the static check errors, add them to git") + console.print( + f" 2) run `breeze testing providers-tests --test-type " + rf"'Providers\[{','.join(provider_ids)}]'` and fix all tests." + ) + console.print(" 3) Add changes to git, create branch, commit the changes and create a PR!") + console.print("\nGood luck!\n") else: console.print("\n[yellow]Dry-run mode, no changes were made.\n") console.print(f"\nThere are currently {count_providers} providers left in the old structure.\n") @@ -209,7 +233,7 @@ def move_providers( def fix_boring_cyborg(provider_id: str): boring_cyborg_file_path = ROOT_PROJECT_DIR_PATH / ".github" / "boring-cyborg.yml" - console.print(f"\n[bright_blue]Updating {boring_cyborg_file_path}\n") + console.rule(f"Updating {boring_cyborg_file_path}", style="bright_blue") original_content = boring_cyborg_file_path.read_text().splitlines() updated_content = [] in_provider = False @@ -228,17 +252,29 @@ def fix_boring_cyborg(provider_id: str): from_content=original_content, updated_content=updated_content, ) + provider_only_path = _get_provider_only_path(provider_id) + _replace_string( + boring_cyborg_file_path, + f"providers/src/airflow/providers/{provider_only_path}", + f"providers/{provider_only_path}/src/airflow/providers/{provider_only_path}", + ) + _replace_string( + boring_cyborg_file_path, + f"providers/tests/{provider_only_path}", + f"providers/{provider_only_path}/tests/provider_tests/{provider_only_path}", + ) + console.rule(style="bright_blue") def add_docs_to_gitignore(provider_id: str): gitignore_path = DOCS_DIR_PATH / ".gitignore" - console.print(f"\n[bright_blue]Updating {gitignore_path}\n") + console.rule(f"Updating {gitignore_path}", style="bright_blue") original_content = gitignore_path.read_text().splitlines() provider_line = f"apache-airflow-providers-{provider_id.replace('.', '-')}" if provider_line in original_content: console.print(f"\n[yellow]Provider {provider_id} already in .gitignore\n") return - updated_content = [] + updated_content: list[str] = [] updated = False for line in original_content: if not line.startswith("#") and line > provider_line and not updated: @@ -253,33 +289,27 @@ def add_docs_to_gitignore(provider_id: str): from_content=original_content, updated_content=updated_content, ) - - -def _replace_string(path: Path, old: str, new: str): - content = path.read_text() - new_content = content.replace(old, new) - if content != new_content: - console.print(f"\n[bright_blue]Replacing {old} with {new} in {path}\n") - if not is_dry_run: - path.write_text(new_content) + console.rule(style="bright_blue") def remove_changelog(provider_id: str): changelog_path = DOCS_DIR_PATH / _get_provider_distribution_name(provider_id) / "changelog.rst" - console.print(f"\n[bright_blue]Deleting {changelog_path}\n") + console.rule(f"Deleting {changelog_path}", style="bright_blue") _do_stuff(syntax="gitignore", from_path=changelog_path, delete_from=True) + console.rule(style="bright_blue") def create_readme(provider_id: str): readme_path = PROVIDERS_DIR_PATH / _get_provider_only_path(provider_id) / "README.rst" - console.print(f"\n[bright_blue]Creating {readme_path}\n") + console.rule(f"Creating {readme_path}", style="bright_blue") _do_stuff(syntax="rst", to_path=readme_path, updated_content=CONTENT_OVERRIDE) + console.rule(style="bright_blue") def move_docs(provider_id: str): source_doc_dir = DOCS_DIR_PATH / _get_provider_distribution_name(provider_id) dest_doc_dir = PROVIDERS_DIR_PATH / _get_provider_only_path(provider_id) / "docs" - console.print(f"\n[bright_blue]Moving docs to {dest_doc_dir}\n") + console.rule(f"Moving docs to {dest_doc_dir}", style="bright_blue") _do_stuff(syntax="rst", from_path=source_doc_dir, to_path=dest_doc_dir, delete_from=True) provider_package_source_dir = OLD_PROVIDERS_AIRFLOW_PROVIDERS_SRC_PACKAGE_PATH / _get_provider_only_path( provider_id @@ -290,12 +320,14 @@ def move_docs(provider_id: str): to_path=dest_doc_dir / "changelog.rst", delete_from=True, ) - _do_stuff( - syntax="txt", - from_path=provider_package_source_dir / ".latest-doc-only-change.txt", - to_path=dest_doc_dir / ".latest-doc-only-change.txt", - delete_from=True, - ) + if (provider_package_source_dir / ".latest-doc-only-change.txt").exists(): + _do_stuff( + syntax="txt", + from_path=provider_package_source_dir / ".latest-doc-only-change.txt", + to_path=dest_doc_dir / ".latest-doc-only-change.txt", + delete_from=True, + ) + console.rule(style="bright_blue") def move_provider_yaml(provider_id: str) -> tuple[list[str], list[str], list[str]]: @@ -305,7 +337,7 @@ def move_provider_yaml(provider_id: str) -> tuple[list[str], list[str], list[str / "provider.yaml" ) target_provider_yaml_path = PROVIDERS_DIR_PATH / _get_provider_only_path(provider_id) / "provider.yaml" - console.print(f"\n[bright_blue]Moving {source_provider_yaml_path} to {target_provider_yaml_path}\n") + console.rule(f"Moving {source_provider_yaml_path} to {target_provider_yaml_path}", style="bright_blue") original_content = source_provider_yaml_path.read_text().splitlines() in_dependencies = False in_optional_dependencies = False @@ -352,7 +384,6 @@ def move_provider_yaml(provider_id: str) -> tuple[list[str], list[str], list[str in_optional_dependencies = False if not in_dependencies and not in_optional_dependencies and not in_devel_dependencies: updated_content.append(line) - _do_stuff( syntax="yml", from_path=source_provider_yaml_path, @@ -384,6 +415,7 @@ def move_provider_yaml(provider_id: str) -> tuple[list[str], list[str], list[str optional_dependencies_processed.append("]") else: optional_dependencies_processed = [] + console.rule(style="bright_blue") return ( dependencies, devel_dependencies, @@ -429,8 +461,8 @@ def create_pyproject_toml( [project.urls] """ pyproject_toml_path = PROVIDERS_DIR_PATH / _get_provider_only_path(provider_id) / "pyproject.toml" - console.print( - f"\n[bright_blue]Creating basic pyproject.toml for {provider_id} in {pyproject_toml_path}\n" + console.rule( + f"Creating basic pyproject.toml for {provider_id} in {pyproject_toml_path}", style="bright_blue" ) pyproject_toml_content = start_pyproject_toml @@ -440,6 +472,7 @@ def create_pyproject_toml( pyproject_toml_content += devel_dependencies_toml _do_stuff(syntax="toml", to_path=pyproject_toml_path, updated_content=pyproject_toml_content.splitlines()) + console.rule(style="bright_blue") def move_sources(provider_id: str): @@ -454,41 +487,119 @@ def move_sources(provider_id: str): / "providers" / _get_provider_only_path(provider_id) ) - console.print(f"\n[bright_blue]Moving sources from {source_provider_dir} to {dest_provider_dir}\n") + console.rule(f"Moving sources from {source_provider_dir} to {dest_provider_dir}", style="bright_blue") _do_stuff(syntax="bash", from_path=source_provider_dir, to_path=dest_provider_dir, delete_from=True) + console.rule(style="bright_blue") def move_tests(provider_id: str): source_test_dir = OLD_PROVIDERS_TEST_DIR_PATH / _get_provider_only_path(provider_id) - dest_test_dir = ( - PROVIDERS_DIR_PATH - / _get_provider_only_path(provider_id) - / "tests" - / "providers" - / _get_provider_only_path(provider_id) - ) - console.print(f"\n[bright_blue]Moving tests from {source_test_dir} to {dest_test_dir}\n") + airflow_tests_dir = ROOT_PROJECT_DIR_PATH / "tests" + root_dest_test_dir = PROVIDERS_DIR_PATH / _get_provider_only_path(provider_id) / "tests" + dest_test_dir = root_dest_test_dir / "provider_tests" / _get_provider_only_path(provider_id) + console.rule(f"Moving tests from {source_test_dir} to {dest_test_dir}", style="bright_blue") + + for test_file_path in source_test_dir.rglob("*.py"): + _replace_string( + test_file_path, f"from providers.tests.{provider_id}", f"from provider_tests.{provider_id}" + ) + for test_file_path in airflow_tests_dir.rglob("*.py"): + _replace_string( + test_file_path, f"from providers.tests.{provider_id}", f"from provider_tests.{provider_id}" + ) _do_stuff(syntax="bash", from_path=source_test_dir, to_path=dest_test_dir, delete_from=True) + conftest_py_content = """ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations -def move_system_tests(provider_id: str): - source_system_test_dir = OLD_PROVIDERS_SYSTEM_TEST_DIR_PATH / _get_provider_only_path(provider_id) - dest_system_test_dir = ( - PROVIDERS_DIR_PATH - / _get_provider_only_path(provider_id) - / "tests" - / "system" - / _get_provider_only_path(provider_id) - ) - console.print( - f"\n[bright_blue]Moving system tests from {source_system_test_dir} to {dest_system_test_dir}\n" +import pathlib + +import pytest + +pytest_plugins = "tests_common.pytest_plugin" + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + deprecations_ignore_path = pathlib.Path(__file__).parent.joinpath("deprecations_ignore.yml") + dep_path = [deprecations_ignore_path] if deprecations_ignore_path.exists() else [] + config.inicfg["airflow_deprecations_ignore"] = ( + config.inicfg.get("airflow_deprecations_ignore", []) + dep_path # type: ignore[assignment,operator] ) - _do_stuff(syntax="bash", from_path=source_system_test_dir, to_path=dest_system_test_dir, delete_from=True) +"""[1:] + + conftest_py_path = root_dest_test_dir / "conftest.py" + _do_stuff(syntax="python", to_path=conftest_py_path, updated_content=conftest_py_content.splitlines()) + init_content = """ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore +"""[1:] + for parent in dest_test_dir.parents: + if parent.name == "tests": + break + init_file = parent / "__init__.py" + if not init_file.exists(): + _do_stuff(syntax="python", to_path=init_file, updated_content=init_content.splitlines()) + + console.rule(style="bright_blue") + + +def move_system_tests(provider_id: str) -> bool: + source_system_test_dir = OLD_PROVIDERS_SYSTEM_TEST_DIR_PATH / _get_provider_only_path(provider_id) + if source_system_test_dir.exists(): + dest_system_test_dir = ( + PROVIDERS_DIR_PATH + / _get_provider_only_path(provider_id) + / "tests" + / "system" + / _get_provider_only_path(provider_id) + ) + console.rule( + f"Moving system tests from {source_system_test_dir} to {dest_system_test_dir}", + style="bright_blue", + ) + _do_stuff( + syntax="bash", from_path=source_system_test_dir, to_path=dest_system_test_dir, delete_from=True + ) + console.rule(style="bright_blue") + return True + return False def replace_system_test_example_includes(provider_id: str): target_doc_providers_dir = PROVIDERS_DIR_PATH / _get_provider_only_path(provider_id) / "docs" - console.print(f"\n[bright_blue]Replacing system test example includes in {target_doc_providers_dir}\n") + console.rule(f"Replacing system test example includes in {target_doc_providers_dir}", style="bright_blue") for rst_file in target_doc_providers_dir.rglob("*.rst"): provider_only_path = _get_provider_only_path(provider_id) _replace_string( @@ -496,10 +607,75 @@ def replace_system_test_example_includes(provider_id: str): f"../providers/tests/system/{provider_only_path}/", f"../providers/{provider_only_path}/tests/system/{provider_only_path}/", ) + console.rule(style="bright_blue") + + +def update_airflow_pyproject_toml(provider_id): + pyproject_toml_path = ROOT_PROJECT_DIR_PATH / "pyproject.toml" + console.rule(f"Updating {pyproject_toml_path}", style="bright_blue") + content = pyproject_toml_path.read_text().splitlines() + updated_content: list[str] = [] + distribution_name = _get_provider_distribution_name(provider_id) + only_provider_path = _get_provider_only_path(provider_id) + in_dependency_groups = False + in_tool_uv_sources = False + in_tool_uv_workspace = False + + dependency_line_to_add = f' "{distribution_name}",' + sources_line_to_add = f"{distribution_name} = {{ workspace = true }}" + workspace_line_to_add = f' "providers/{only_provider_path}",' + + for line in content: + if line.startswith("[dependency-groups]"): + in_dependency_groups = True + elif in_dependency_groups and ( + line.startswith("]") or (line.startswith(' "apache') and dependency_line_to_add < line) + ): + updated_content.append(dependency_line_to_add) + in_dependency_groups = False + if line.startswith("[tool.uv.sources]"): + in_tool_uv_sources = True + elif in_tool_uv_sources and ( + line.strip() == "" + or line.startswith("[") + or (line.startswith("apache") and sources_line_to_add < line) + ): + updated_content.append(sources_line_to_add) + in_tool_uv_sources = False + if line.startswith("[tool.uv.workspace]"): + in_tool_uv_workspace = True + elif in_tool_uv_workspace and ( + line.startswith("]") or (line.startswith(" ") and workspace_line_to_add < line) + ): + updated_content.append(workspace_line_to_add) + in_tool_uv_workspace = False + updated_content.append(line) + _do_stuff( + syntax="toml", + from_path=pyproject_toml_path, + from_content=content, + updated_content=updated_content, + ) + console.rule(style="bright_blue") + + +def fix_selective_checks_test(provider_id: str): + selective_checks_test_path = ( + ROOT_PROJECT_DIR_PATH / "dev" / "breeze" / "tests" / "test_selective_checks.py" + ) + provider_only_path = _get_provider_only_path(provider_id) + console.rule(f"Updating {selective_checks_test_path}", style="bright_blue") + _replace_string( + selective_checks_test_path, + f"providers/src/airflow/providers/{provider_only_path}", + f"providers/{provider_only_path}/src/airflow/providers/{provider_only_path}", + ) def move_provider(provider_id: str): + fix_selective_checks_test(provider_id) fix_boring_cyborg(provider_id) + update_airflow_pyproject_toml(provider_id) add_docs_to_gitignore(provider_id) remove_changelog(provider_id) create_readme(provider_id) @@ -508,8 +684,9 @@ def move_provider(provider_id: str): create_pyproject_toml(provider_id, dependencies, devel_dependencies, optional_dependencies) move_sources(provider_id) move_tests(provider_id) - move_system_tests(provider_id) - replace_system_test_example_includes(provider_id) + has_system_test = move_system_tests(provider_id) + if has_system_test: + replace_system_test_example_includes(provider_id) if __name__ == "__main__": diff --git a/docs/.gitignore b/docs/.gitignore index 97a0827b426a38..d05383991a9d94 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,6 +1,8 @@ # TODO(potiuk): change it to apache-airflow-providers-* after all providers are migrated to the new structure # Eventually when we swtich to individually build docs for each provider, we should remove this altogether apache-airflow-providers-airbyte +apache-airflow-providers-alibaba apache-airflow-providers-apache-iceberg apache-airflow-providers-celery apache-airflow-providers-edge +apache-airflow-providers-standard diff --git a/docs/apache-airflow-providers-alibaba/changelog.rst b/docs/apache-airflow-providers-alibaba/changelog.rst deleted file mode 100644 index f4c779dbf50119..00000000000000 --- a/docs/apache-airflow-providers-alibaba/changelog.rst +++ /dev/null @@ -1,25 +0,0 @@ - - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE - OVERWRITTEN WHEN PREPARING PACKAGES. - - .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE - `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY - -.. include:: ../../providers/src/airflow/providers/alibaba/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-standard/changelog.rst b/docs/apache-airflow-providers-standard/changelog.rst deleted file mode 100644 index b3700093f417f3..00000000000000 --- a/docs/apache-airflow-providers-standard/changelog.rst +++ /dev/null @@ -1,25 +0,0 @@ - - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE - OVERWRITTEN WHEN PREPARING PACKAGES. - - .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE - `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY - -.. include:: ../../providers/src/airflow/providers/standard/CHANGELOG.rst diff --git a/providers/airbyte/pyproject.toml b/providers/airbyte/pyproject.toml index 37022d69b5ced0..4f18caaf980006 100644 --- a/providers/airbyte/pyproject.toml +++ b/providers/airbyte/pyproject.toml @@ -43,7 +43,12 @@ classifiers = [ "Intended Audience :: System Administrators", "Framework :: Apache Airflow", "Framework :: Apache Airflow :: Provider", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Topic :: System :: Monitoring", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", ] requires-python = "~=3.9" diff --git a/providers/airbyte/tests/provider_tests/__init__.py b/providers/airbyte/tests/provider_tests/__init__.py index 31d3edfae031a9..e8fd22856438c4 100644 --- a/providers/airbyte/tests/provider_tests/__init__.py +++ b/providers/airbyte/tests/provider_tests/__init__.py @@ -14,6 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/alibaba/README.rst b/providers/alibaba/README.rst new file mode 100644 index 00000000000000..de14ee05fd541c --- /dev/null +++ b/providers/alibaba/README.rst @@ -0,0 +1,64 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +Package ``apache-airflow-providers-alibaba`` + +Release: ``3.0.0`` + + +Alibaba Cloud integration (including `Alibaba Cloud `__). + + +Provider package +---------------- + +This is a provider package for ``alibaba`` provider. All classes for this provider package +are in ``airflow.providers.alibaba`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-alibaba`` + +The package supports the following python versions: 3.9,3.10,3.11,3.12 + +Requirements +------------ + +============================ ================== +PIP package Version required +============================ ================== +``apache-airflow`` ``>=2.9.0`` +``oss2`` ``>=2.14.0`` +``alibabacloud_adb20211201`` ``>=1.0.0`` +``alibabacloud_tea_openapi`` ``>=0.3.7`` +============================ ================== + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/src/airflow/providers/alibaba/.latest-doc-only-change.txt b/providers/alibaba/docs/.latest-doc-only-change.txt similarity index 100% rename from providers/src/airflow/providers/alibaba/.latest-doc-only-change.txt rename to providers/alibaba/docs/.latest-doc-only-change.txt diff --git a/providers/src/airflow/providers/alibaba/CHANGELOG.rst b/providers/alibaba/docs/changelog.rst similarity index 100% rename from providers/src/airflow/providers/alibaba/CHANGELOG.rst rename to providers/alibaba/docs/changelog.rst diff --git a/docs/apache-airflow-providers-alibaba/commits.rst b/providers/alibaba/docs/commits.rst similarity index 100% rename from docs/apache-airflow-providers-alibaba/commits.rst rename to providers/alibaba/docs/commits.rst diff --git a/docs/apache-airflow-providers-alibaba/connections/alibaba.rst b/providers/alibaba/docs/connections/alibaba.rst similarity index 100% rename from docs/apache-airflow-providers-alibaba/connections/alibaba.rst rename to providers/alibaba/docs/connections/alibaba.rst diff --git a/docs/apache-airflow-providers-alibaba/index.rst b/providers/alibaba/docs/index.rst similarity index 100% rename from docs/apache-airflow-providers-alibaba/index.rst rename to providers/alibaba/docs/index.rst diff --git a/docs/apache-airflow-providers-alibaba/installing-providers-from-sources.rst b/providers/alibaba/docs/installing-providers-from-sources.rst similarity index 100% rename from docs/apache-airflow-providers-alibaba/installing-providers-from-sources.rst rename to providers/alibaba/docs/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-alibaba/logging/index.rst b/providers/alibaba/docs/logging/index.rst similarity index 100% rename from docs/apache-airflow-providers-alibaba/logging/index.rst rename to providers/alibaba/docs/logging/index.rst diff --git a/docs/apache-airflow-providers-alibaba/logging/oss-task-handler.rst b/providers/alibaba/docs/logging/oss-task-handler.rst similarity index 100% rename from docs/apache-airflow-providers-alibaba/logging/oss-task-handler.rst rename to providers/alibaba/docs/logging/oss-task-handler.rst diff --git a/docs/apache-airflow-providers-alibaba/operators/analyticdb_spark.rst b/providers/alibaba/docs/operators/analyticdb_spark.rst similarity index 94% rename from docs/apache-airflow-providers-alibaba/operators/analyticdb_spark.rst rename to providers/alibaba/docs/operators/analyticdb_spark.rst index 952eb6c2c60373..563c91f77c958c 100644 --- a/docs/apache-airflow-providers-alibaba/operators/analyticdb_spark.rst +++ b/providers/alibaba/docs/operators/analyticdb_spark.rst @@ -39,7 +39,7 @@ Defining tasks In the following code we submit Spark Pi and Spark Logistic regression applications. -.. exampleinclude:: /../../providers/tests/system/alibaba/example_adb_spark_batch.py +.. exampleinclude:: /../../providers/alibaba/tests/system/alibaba/example_adb_spark_batch.py :language: python :start-after: [START howto_operator_adb_spark_batch] :end-before: [END howto_operator_adb_spark_batch] diff --git a/docs/apache-airflow-providers-alibaba/operators/index.rst b/providers/alibaba/docs/operators/index.rst similarity index 100% rename from docs/apache-airflow-providers-alibaba/operators/index.rst rename to providers/alibaba/docs/operators/index.rst diff --git a/docs/apache-airflow-providers-alibaba/operators/oss.rst b/providers/alibaba/docs/operators/oss.rst similarity index 95% rename from docs/apache-airflow-providers-alibaba/operators/oss.rst rename to providers/alibaba/docs/operators/oss.rst index 17d3829e374272..7470da0b3b6ab7 100644 --- a/docs/apache-airflow-providers-alibaba/operators/oss.rst +++ b/providers/alibaba/docs/operators/oss.rst @@ -45,7 +45,7 @@ Defining tasks In the following code we create a new bucket and then delete the bucket. -.. exampleinclude:: /../../providers/tests/system/alibaba/example_oss_bucket.py +.. exampleinclude:: /../../providers/alibaba/tests/system/alibaba/example_oss_bucket.py :language: python :start-after: [START howto_operator_oss_bucket] :end-before: [END howto_operator_oss_bucket] diff --git a/docs/apache-airflow-providers-alibaba/redirects.txt b/providers/alibaba/docs/redirects.txt similarity index 100% rename from docs/apache-airflow-providers-alibaba/redirects.txt rename to providers/alibaba/docs/redirects.txt diff --git a/docs/apache-airflow-providers-alibaba/security.rst b/providers/alibaba/docs/security.rst similarity index 100% rename from docs/apache-airflow-providers-alibaba/security.rst rename to providers/alibaba/docs/security.rst diff --git a/providers/src/airflow/providers/alibaba/provider.yaml b/providers/alibaba/provider.yaml similarity index 95% rename from providers/src/airflow/providers/alibaba/provider.yaml rename to providers/alibaba/provider.yaml index a5d8b54e400f04..0069e9b931d1d8 100644 --- a/providers/src/airflow/providers/alibaba/provider.yaml +++ b/providers/alibaba/provider.yaml @@ -51,12 +51,6 @@ versions: - 1.0.1 - 1.0.0 -dependencies: - - apache-airflow>=2.9.0 - - oss2>=2.14.0 - - alibabacloud_adb20211201>=1.0.0 - - alibabacloud_tea_openapi>=0.3.7 - integrations: - integration-name: Alibaba Cloud OSS external-doc-url: https://www.alibabacloud.com/help/product/31815.htm diff --git a/providers/alibaba/pyproject.toml b/providers/alibaba/pyproject.toml new file mode 100644 index 00000000000000..f25407832261af --- /dev/null +++ b/providers/alibaba/pyproject.toml @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.10.1"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-alibaba" +version = "3.0.0" +description = "Provider package apache-airflow-providers-alibaba for Apache Airflow" +readme = "README.rst" +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "alibaba", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", +] +requires-python = "~=3.9" + +# The dependencies should be modified in place in the generated file +# Any change in the dependencies is preserved when the file is regenerated +dependencies = [ + "apache-airflow>=2.9.0", + "oss2>=2.14.0", + "alibabacloud_adb20211201>=1.0.0", + "alibabacloud_tea_openapi>=0.3.7", +] + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-alibaba/3.0.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-alibaba/3.0.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Twitter" = "https://x.com/ApacheAirflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.alibaba.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.alibaba" + +[tool.pytest.ini_options] +ignore = "tests/system/" diff --git a/providers/alibaba/src/airflow/providers/alibaba/LICENSE b/providers/alibaba/src/airflow/providers/alibaba/LICENSE new file mode 100644 index 00000000000000..11069edd79019f --- /dev/null +++ b/providers/alibaba/src/airflow/providers/alibaba/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/src/airflow/providers/alibaba/__init__.py b/providers/alibaba/src/airflow/providers/alibaba/__init__.py similarity index 100% rename from providers/src/airflow/providers/alibaba/__init__.py rename to providers/alibaba/src/airflow/providers/alibaba/__init__.py diff --git a/providers/src/airflow/providers/alibaba/cloud/__init__.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/__init__.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/__init__.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/__init__.py diff --git a/providers/src/airflow/providers/alibaba/cloud/hooks/__init__.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/hooks/__init__.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/hooks/__init__.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/hooks/__init__.py diff --git a/providers/src/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py diff --git a/providers/src/airflow/providers/alibaba/cloud/hooks/oss.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/hooks/oss.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/hooks/oss.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/hooks/oss.py diff --git a/providers/src/airflow/providers/alibaba/cloud/log/__init__.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/log/__init__.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/log/__init__.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/log/__init__.py diff --git a/providers/src/airflow/providers/alibaba/cloud/log/oss_task_handler.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/log/oss_task_handler.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/log/oss_task_handler.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/log/oss_task_handler.py diff --git a/providers/src/airflow/providers/alibaba/cloud/operators/__init__.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/operators/__init__.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/operators/__init__.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/operators/__init__.py diff --git a/providers/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py diff --git a/providers/src/airflow/providers/alibaba/cloud/operators/oss.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/operators/oss.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/operators/oss.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/operators/oss.py diff --git a/providers/src/airflow/providers/alibaba/cloud/sensors/__init__.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/sensors/__init__.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/sensors/__init__.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/sensors/__init__.py diff --git a/providers/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py diff --git a/providers/src/airflow/providers/alibaba/cloud/sensors/oss_key.py b/providers/alibaba/src/airflow/providers/alibaba/cloud/sensors/oss_key.py similarity index 100% rename from providers/src/airflow/providers/alibaba/cloud/sensors/oss_key.py rename to providers/alibaba/src/airflow/providers/alibaba/cloud/sensors/oss_key.py diff --git a/providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py b/providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py new file mode 100644 index 00000000000000..c253ea090beb63 --- /dev/null +++ b/providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-alibaba", + "name": "Alibaba", + "description": "Alibaba Cloud integration (including `Alibaba Cloud `__).\n", + "state": "ready", + "source-date-epoch": 1734526745, + "versions": [ + "3.0.0", + "2.9.1", + "2.9.0", + "2.8.1", + "2.8.0", + "2.7.3", + "2.7.2", + "2.7.1", + "2.7.0", + "2.6.0", + "2.5.3", + "2.5.2", + "2.5.1", + "2.5.0", + "2.4.1", + "2.4.0", + "2.3.0", + "2.2.0", + "2.1.0", + "2.0.1", + "2.0.0", + "1.1.1", + "1.1.0", + "1.0.1", + "1.0.0", + ], + "integrations": [ + { + "integration-name": "Alibaba Cloud OSS", + "external-doc-url": "https://www.alibabacloud.com/help/product/31815.htm", + "logo": "/integration-logos/alibaba/cloud/alibabacloud-oss.png", + "how-to-guide": ["/docs/apache-airflow-providers-alibaba/operators/oss.rst"], + "tags": ["alibaba"], + }, + { + "integration-name": "Alibaba Cloud AnalyticDB Spark", + "external-doc-url": "https://www.alibabacloud.com/help/en/analyticdb-for-mysql/latest/spark-developerment", + "how-to-guide": ["/docs/apache-airflow-providers-alibaba/operators/analyticdb_spark.rst"], + "tags": ["alibaba"], + }, + ], + "operators": [ + { + "integration-name": "Alibaba Cloud OSS", + "python-modules": ["airflow.providers.alibaba.cloud.operators.oss"], + }, + { + "integration-name": "Alibaba Cloud AnalyticDB Spark", + "python-modules": ["airflow.providers.alibaba.cloud.operators.analyticdb_spark"], + }, + ], + "sensors": [ + { + "integration-name": "Alibaba Cloud OSS", + "python-modules": ["airflow.providers.alibaba.cloud.sensors.oss_key"], + }, + { + "integration-name": "Alibaba Cloud AnalyticDB Spark", + "python-modules": ["airflow.providers.alibaba.cloud.sensors.analyticdb_spark"], + }, + ], + "hooks": [ + { + "integration-name": "Alibaba Cloud OSS", + "python-modules": ["airflow.providers.alibaba.cloud.hooks.oss"], + }, + { + "integration-name": "Alibaba Cloud AnalyticDB Spark", + "python-modules": ["airflow.providers.alibaba.cloud.hooks.analyticdb_spark"], + }, + ], + "connection-types": [ + { + "hook-class-name": "airflow.providers.alibaba.cloud.hooks.oss.OSSHook", + "connection-type": "oss", + }, + { + "hook-class-name": "airflow.providers.alibaba.cloud.hooks.analyticdb_spark.AnalyticDBSparkHook", + "connection-type": "adb_spark", + }, + ], + "logging": ["airflow.providers.alibaba.cloud.log.oss_task_handler.OSSTaskHandler"], + "dependencies": [ + "apache-airflow>=2.9.0", + "oss2>=2.14.0", + "alibabacloud_adb20211201>=1.0.0", + "alibabacloud_tea_openapi>=0.3.7", + ], + } diff --git a/providers/alibaba/tests/conftest.py b/providers/alibaba/tests/conftest.py new file mode 100644 index 00000000000000..068fe6bbf5ae9a --- /dev/null +++ b/providers/alibaba/tests/conftest.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pathlib + +import pytest + +pytest_plugins = "tests_common.pytest_plugin" + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + deprecations_ignore_path = pathlib.Path(__file__).parent.joinpath("deprecations_ignore.yml") + dep_path = [deprecations_ignore_path] if deprecations_ignore_path.exists() else [] + config.inicfg["airflow_deprecations_ignore"] = ( + config.inicfg.get("airflow_deprecations_ignore", []) + dep_path # type: ignore[assignment,operator] + ) diff --git a/providers/alibaba/tests/provider_tests/__init__.py b/providers/alibaba/tests/provider_tests/__init__.py new file mode 100644 index 00000000000000..e8fd22856438c4 --- /dev/null +++ b/providers/alibaba/tests/provider_tests/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/src/airflow/providers/standard/hooks/__init__.py b/providers/alibaba/tests/provider_tests/alibaba/__init__.py similarity index 100% rename from providers/src/airflow/providers/standard/hooks/__init__.py rename to providers/alibaba/tests/provider_tests/alibaba/__init__.py diff --git a/providers/src/airflow/providers/standard/operators/__init__.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/__init__.py similarity index 100% rename from providers/src/airflow/providers/standard/operators/__init__.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/__init__.py diff --git a/providers/src/airflow/providers/standard/sensors/__init__.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/hooks/__init__.py similarity index 100% rename from providers/src/airflow/providers/standard/sensors/__init__.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/hooks/__init__.py diff --git a/providers/tests/alibaba/cloud/hooks/test_analyticdb_spark.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/hooks/test_analyticdb_spark.py similarity index 98% rename from providers/tests/alibaba/cloud/hooks/test_analyticdb_spark.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/hooks/test_analyticdb_spark.py index b3e9100e2e59d3..ac64be058890e1 100644 --- a/providers/tests/alibaba/cloud/hooks/test_analyticdb_spark.py +++ b/providers/alibaba/tests/provider_tests/alibaba/cloud/hooks/test_analyticdb_spark.py @@ -34,8 +34,7 @@ ) from airflow.providers.alibaba.cloud.hooks.analyticdb_spark import AnalyticDBSparkHook - -from providers.tests.alibaba.cloud.utils.analyticdb_spark_mock import mock_adb_spark_hook_default_project_id +from provider_tests.alibaba.cloud.utils.analyticdb_spark_mock import mock_adb_spark_hook_default_project_id ADB_SPARK_STRING = "airflow.providers.alibaba.cloud.hooks.analyticdb_spark.{}" MOCK_ADB_SPARK_CONN_ID = "mock_id" diff --git a/providers/tests/alibaba/cloud/hooks/test_oss.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/hooks/test_oss.py similarity index 98% rename from providers/tests/alibaba/cloud/hooks/test_oss.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/hooks/test_oss.py index 2bb53dddece5cc..edb1a7f276da6d 100644 --- a/providers/tests/alibaba/cloud/hooks/test_oss.py +++ b/providers/alibaba/tests/provider_tests/alibaba/cloud/hooks/test_oss.py @@ -20,8 +20,7 @@ from unittest import mock from airflow.providers.alibaba.cloud.hooks.oss import OSSHook - -from providers.tests.alibaba.cloud.utils.oss_mock import mock_oss_hook_default_project_id +from provider_tests.alibaba.cloud.utils.oss_mock import mock_oss_hook_default_project_id OSS_STRING = "airflow.providers.alibaba.cloud.hooks.oss.{}" MOCK_OSS_CONN_ID = "mock_id" diff --git a/providers/src/airflow/providers/standard/triggers/__init__.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/log/__init__.py similarity index 100% rename from providers/src/airflow/providers/standard/triggers/__init__.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/log/__init__.py diff --git a/providers/tests/alibaba/cloud/log/test_oss_task_handler.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/log/test_oss_task_handler.py similarity index 100% rename from providers/tests/alibaba/cloud/log/test_oss_task_handler.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/log/test_oss_task_handler.py diff --git a/providers/src/airflow/providers/standard/utils/__init__.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/operators/__init__.py similarity index 100% rename from providers/src/airflow/providers/standard/utils/__init__.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/operators/__init__.py diff --git a/providers/tests/alibaba/cloud/operators/test_analyticdb_spark.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/operators/test_analyticdb_spark.py similarity index 100% rename from providers/tests/alibaba/cloud/operators/test_analyticdb_spark.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/operators/test_analyticdb_spark.py diff --git a/providers/tests/alibaba/cloud/operators/test_oss.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/operators/test_oss.py similarity index 100% rename from providers/tests/alibaba/cloud/operators/test_oss.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/operators/test_oss.py diff --git a/providers/tests/alibaba/__init__.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/sensors/__init__.py similarity index 100% rename from providers/tests/alibaba/__init__.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/sensors/__init__.py diff --git a/providers/tests/alibaba/cloud/sensors/test_analyticdb_spark.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/sensors/test_analyticdb_spark.py similarity index 100% rename from providers/tests/alibaba/cloud/sensors/test_analyticdb_spark.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/sensors/test_analyticdb_spark.py diff --git a/providers/tests/alibaba/cloud/sensors/test_oss_key.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/sensors/test_oss_key.py similarity index 100% rename from providers/tests/alibaba/cloud/sensors/test_oss_key.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/sensors/test_oss_key.py diff --git a/providers/tests/alibaba/cloud/__init__.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/utils/__init__.py similarity index 100% rename from providers/tests/alibaba/cloud/__init__.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/utils/__init__.py diff --git a/providers/tests/alibaba/cloud/utils/analyticdb_spark_mock.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/utils/analyticdb_spark_mock.py similarity index 100% rename from providers/tests/alibaba/cloud/utils/analyticdb_spark_mock.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/utils/analyticdb_spark_mock.py diff --git a/providers/tests/alibaba/cloud/utils/oss_mock.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/utils/oss_mock.py similarity index 100% rename from providers/tests/alibaba/cloud/utils/oss_mock.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/utils/oss_mock.py diff --git a/providers/tests/alibaba/cloud/utils/test_utils.py b/providers/alibaba/tests/provider_tests/alibaba/cloud/utils/test_utils.py similarity index 100% rename from providers/tests/alibaba/cloud/utils/test_utils.py rename to providers/alibaba/tests/provider_tests/alibaba/cloud/utils/test_utils.py diff --git a/providers/tests/alibaba/cloud/hooks/__init__.py b/providers/alibaba/tests/system/alibaba/__init__.py similarity index 100% rename from providers/tests/alibaba/cloud/hooks/__init__.py rename to providers/alibaba/tests/system/alibaba/__init__.py diff --git a/providers/tests/system/alibaba/example_adb_spark_batch.py b/providers/alibaba/tests/system/alibaba/example_adb_spark_batch.py similarity index 100% rename from providers/tests/system/alibaba/example_adb_spark_batch.py rename to providers/alibaba/tests/system/alibaba/example_adb_spark_batch.py diff --git a/providers/tests/system/alibaba/example_adb_spark_sql.py b/providers/alibaba/tests/system/alibaba/example_adb_spark_sql.py similarity index 100% rename from providers/tests/system/alibaba/example_adb_spark_sql.py rename to providers/alibaba/tests/system/alibaba/example_adb_spark_sql.py diff --git a/providers/tests/system/alibaba/example_oss_bucket.py b/providers/alibaba/tests/system/alibaba/example_oss_bucket.py similarity index 100% rename from providers/tests/system/alibaba/example_oss_bucket.py rename to providers/alibaba/tests/system/alibaba/example_oss_bucket.py diff --git a/providers/tests/system/alibaba/example_oss_object.py b/providers/alibaba/tests/system/alibaba/example_oss_object.py similarity index 100% rename from providers/tests/system/alibaba/example_oss_object.py rename to providers/alibaba/tests/system/alibaba/example_oss_object.py diff --git a/providers/apache/iceberg/pyproject.toml b/providers/apache/iceberg/pyproject.toml index 8059b1b554dedf..4b70aeb9228fd8 100644 --- a/providers/apache/iceberg/pyproject.toml +++ b/providers/apache/iceberg/pyproject.toml @@ -43,7 +43,12 @@ classifiers = [ "Intended Audience :: System Administrators", "Framework :: Apache Airflow", "Framework :: Apache Airflow :: Provider", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Topic :: System :: Monitoring", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", ] requires-python = "~=3.9" diff --git a/providers/apache/iceberg/tests/provider_tests/__init__.py b/providers/apache/iceberg/tests/provider_tests/__init__.py index 31d3edfae031a9..e8fd22856438c4 100644 --- a/providers/apache/iceberg/tests/provider_tests/__init__.py +++ b/providers/apache/iceberg/tests/provider_tests/__init__.py @@ -14,6 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/iceberg/tests/provider_tests/apache/__init__.py b/providers/apache/iceberg/tests/provider_tests/apache/__init__.py index 31d3edfae031a9..e8fd22856438c4 100644 --- a/providers/apache/iceberg/tests/provider_tests/apache/__init__.py +++ b/providers/apache/iceberg/tests/provider_tests/apache/__init__.py @@ -14,6 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/celery/pyproject.toml b/providers/celery/pyproject.toml index bbba268f753a5b..011a3812c4803a 100644 --- a/providers/celery/pyproject.toml +++ b/providers/celery/pyproject.toml @@ -43,7 +43,12 @@ classifiers = [ "Intended Audience :: System Administrators", "Framework :: Apache Airflow", "Framework :: Apache Airflow :: Provider", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Topic :: System :: Monitoring", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", ] requires-python = "~=3.9" diff --git a/providers/celery/tests/provider_tests/__init__.py b/providers/celery/tests/provider_tests/__init__.py index 31d3edfae031a9..e8fd22856438c4 100644 --- a/providers/celery/tests/provider_tests/__init__.py +++ b/providers/celery/tests/provider_tests/__init__.py @@ -14,6 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/edge/pyproject.toml b/providers/edge/pyproject.toml index 055e38ec525a33..9b1d18d286cc09 100644 --- a/providers/edge/pyproject.toml +++ b/providers/edge/pyproject.toml @@ -43,7 +43,12 @@ classifiers = [ "Intended Audience :: System Administrators", "Framework :: Apache Airflow", "Framework :: Apache Airflow :: Provider", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Topic :: System :: Monitoring", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", ] requires-python = "~=3.9" diff --git a/providers/edge/tests/provider_tests/__init__.py b/providers/edge/tests/provider_tests/__init__.py index 31d3edfae031a9..e8fd22856438c4 100644 --- a/providers/edge/tests/provider_tests/__init__.py +++ b/providers/edge/tests/provider_tests/__init__.py @@ -14,6 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/standard/README.rst b/providers/standard/README.rst new file mode 100644 index 00000000000000..3ad4b01bd8773d --- /dev/null +++ b/providers/standard/README.rst @@ -0,0 +1,62 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +Package ``apache-airflow-providers-standard`` + +Release: ``0.0.3`` + + +Airflow Standard Provider + + +Provider package +---------------- + +This is a provider package for ``standard`` provider. All classes for this provider package +are in ``airflow.providers.standard`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-standard`` + +The package supports the following python versions: 3.9,3.10,3.11,3.12 + +Requirements +------------ + +======================================= ================== +PIP package Version required +======================================= ================== +``apache-airflow`` ``>=2.9.0`` +``apache-airflow-providers-common-sql`` ``>=1.20.0`` +======================================= ================== + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/src/airflow/providers/standard/CHANGELOG.rst b/providers/standard/docs/changelog.rst similarity index 100% rename from providers/src/airflow/providers/standard/CHANGELOG.rst rename to providers/standard/docs/changelog.rst diff --git a/docs/apache-airflow-providers-standard/commits.rst b/providers/standard/docs/commits.rst similarity index 100% rename from docs/apache-airflow-providers-standard/commits.rst rename to providers/standard/docs/commits.rst diff --git a/docs/apache-airflow-providers-standard/configurations-ref.rst b/providers/standard/docs/configurations-ref.rst similarity index 100% rename from docs/apache-airflow-providers-standard/configurations-ref.rst rename to providers/standard/docs/configurations-ref.rst diff --git a/docs/apache-airflow-providers-standard/index.rst b/providers/standard/docs/index.rst similarity index 100% rename from docs/apache-airflow-providers-standard/index.rst rename to providers/standard/docs/index.rst diff --git a/docs/apache-airflow-providers-standard/installing-providers-from-sources.rst b/providers/standard/docs/installing-providers-from-sources.rst similarity index 100% rename from docs/apache-airflow-providers-standard/installing-providers-from-sources.rst rename to providers/standard/docs/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-standard/operators/bash.rst b/providers/standard/docs/operators/bash.rst similarity index 100% rename from docs/apache-airflow-providers-standard/operators/bash.rst rename to providers/standard/docs/operators/bash.rst diff --git a/docs/apache-airflow-providers-standard/operators/datetime.rst b/providers/standard/docs/operators/datetime.rst similarity index 100% rename from docs/apache-airflow-providers-standard/operators/datetime.rst rename to providers/standard/docs/operators/datetime.rst diff --git a/docs/apache-airflow-providers-standard/operators/index.rst b/providers/standard/docs/operators/index.rst similarity index 100% rename from docs/apache-airflow-providers-standard/operators/index.rst rename to providers/standard/docs/operators/index.rst diff --git a/docs/apache-airflow-providers-standard/operators/python.rst b/providers/standard/docs/operators/python.rst similarity index 100% rename from docs/apache-airflow-providers-standard/operators/python.rst rename to providers/standard/docs/operators/python.rst diff --git a/docs/apache-airflow-providers-standard/security.rst b/providers/standard/docs/security.rst similarity index 100% rename from docs/apache-airflow-providers-standard/security.rst rename to providers/standard/docs/security.rst diff --git a/docs/apache-airflow-providers-standard/sensors/bash.rst b/providers/standard/docs/sensors/bash.rst similarity index 100% rename from docs/apache-airflow-providers-standard/sensors/bash.rst rename to providers/standard/docs/sensors/bash.rst diff --git a/docs/apache-airflow-providers-standard/sensors/datetime.rst b/providers/standard/docs/sensors/datetime.rst similarity index 100% rename from docs/apache-airflow-providers-standard/sensors/datetime.rst rename to providers/standard/docs/sensors/datetime.rst diff --git a/docs/apache-airflow-providers-standard/sensors/external_task_sensor.rst b/providers/standard/docs/sensors/external_task_sensor.rst similarity index 100% rename from docs/apache-airflow-providers-standard/sensors/external_task_sensor.rst rename to providers/standard/docs/sensors/external_task_sensor.rst diff --git a/docs/apache-airflow-providers-standard/sensors/file.rst b/providers/standard/docs/sensors/file.rst similarity index 100% rename from docs/apache-airflow-providers-standard/sensors/file.rst rename to providers/standard/docs/sensors/file.rst diff --git a/docs/apache-airflow-providers-standard/sensors/index.rst b/providers/standard/docs/sensors/index.rst similarity index 100% rename from docs/apache-airflow-providers-standard/sensors/index.rst rename to providers/standard/docs/sensors/index.rst diff --git a/docs/apache-airflow-providers-standard/sensors/python.rst b/providers/standard/docs/sensors/python.rst similarity index 100% rename from docs/apache-airflow-providers-standard/sensors/python.rst rename to providers/standard/docs/sensors/python.rst diff --git a/providers/src/airflow/providers/standard/provider.yaml b/providers/standard/provider.yaml similarity index 97% rename from providers/src/airflow/providers/standard/provider.yaml rename to providers/standard/provider.yaml index 0915b96cf7520b..e5693cdc58c507 100644 --- a/providers/src/airflow/providers/standard/provider.yaml +++ b/providers/standard/provider.yaml @@ -28,10 +28,6 @@ versions: - 0.0.2 - 0.0.1 -dependencies: - - apache-airflow>=2.9.0 - - apache-airflow-providers-common-sql>=1.20.0 - integrations: - integration-name: Standard external-doc-url: https://airflow.apache.org/ diff --git a/providers/standard/pyproject.toml b/providers/standard/pyproject.toml new file mode 100644 index 00000000000000..1d888d7c1f7e77 --- /dev/null +++ b/providers/standard/pyproject.toml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.10.1"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-standard" +version = "0.0.3" +description = "Provider package apache-airflow-providers-standard for Apache Airflow" +readme = "README.rst" +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "standard", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", +] +requires-python = "~=3.9" + +# The dependencies should be modified in place in the generated file +# Any change in the dependencies is preserved when the file is regenerated +dependencies = [ + "apache-airflow>=2.9.0", + "apache-airflow-providers-common-sql>=1.20.0", +] + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.3" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-standard/0.0.3/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Twitter" = "https://x.com/ApacheAirflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.standard.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.standard" + +[tool.pytest.ini_options] +ignore = "tests/system/" diff --git a/providers/standard/src/airflow/providers/standard/LICENSE b/providers/standard/src/airflow/providers/standard/LICENSE new file mode 100644 index 00000000000000..11069edd79019f --- /dev/null +++ b/providers/standard/src/airflow/providers/standard/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/src/airflow/providers/standard/__init__.py b/providers/standard/src/airflow/providers/standard/__init__.py similarity index 100% rename from providers/src/airflow/providers/standard/__init__.py rename to providers/standard/src/airflow/providers/standard/__init__.py diff --git a/providers/standard/src/airflow/providers/standard/get_provider_info.py b/providers/standard/src/airflow/providers/standard/get_provider_info.py new file mode 100644 index 00000000000000..a176fa6cb37bc1 --- /dev/null +++ b/providers/standard/src/airflow/providers/standard/get_provider_info.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-standard", + "name": "Standard", + "description": "Airflow Standard Provider\n", + "state": "ready", + "source-date-epoch": 1734536895, + "versions": ["0.0.3", "0.0.2", "0.0.1"], + "integrations": [ + { + "integration-name": "Standard", + "external-doc-url": "https://airflow.apache.org/", + "tags": ["apache"], + "how-to-guide": [ + "/docs/apache-airflow-providers-standard/operators/bash.rst", + "/docs/apache-airflow-providers-standard/operators/python.rst", + "/docs/apache-airflow-providers-standard/operators/datetime.rst", + ], + } + ], + "operators": [ + { + "integration-name": "Standard", + "python-modules": [ + "airflow.providers.standard.operators.datetime", + "airflow.providers.standard.operators.weekday", + "airflow.providers.standard.operators.bash", + "airflow.providers.standard.operators.python", + "airflow.providers.standard.operators.generic_transfer", + "airflow.providers.standard.operators.trigger_dagrun", + "airflow.providers.standard.operators.latest_only", + ], + } + ], + "sensors": [ + { + "integration-name": "Standard", + "python-modules": [ + "airflow.providers.standard.sensors.date_time", + "airflow.providers.standard.sensors.time_delta", + "airflow.providers.standard.sensors.time", + "airflow.providers.standard.sensors.weekday", + "airflow.providers.standard.sensors.bash", + "airflow.providers.standard.sensors.python", + "airflow.providers.standard.sensors.filesystem", + "airflow.providers.standard.sensors.external_task", + ], + } + ], + "hooks": [ + { + "integration-name": "Standard", + "python-modules": [ + "airflow.providers.standard.hooks.filesystem", + "airflow.providers.standard.hooks.package_index", + "airflow.providers.standard.hooks.subprocess", + ], + } + ], + "triggers": [ + { + "integration-name": "Standard", + "python-modules": [ + "airflow.providers.standard.triggers.external_task", + "airflow.providers.standard.triggers.file", + "airflow.providers.standard.triggers.temporal", + ], + } + ], + "config": { + "standard": { + "description": "Options for the standard provider operators.", + "options": { + "venv_install_method": { + "description": "Which python tooling should be used to install the virtual environment.\n\nThe following options are available:\n- ``auto``: Automatically select, use ``uv`` if available, otherwise use ``pip``.\n- ``pip``: Use pip to install the virtual environment.\n- ``uv``: Use uv to install the virtual environment. Must be available in environment PATH.\n", + "version_added": None, + "type": "string", + "example": "uv", + "default": "auto", + } + }, + } + }, + "dependencies": ["apache-airflow>=2.9.0", "apache-airflow-providers-common-sql>=1.20.0"], + } diff --git a/providers/tests/alibaba/cloud/log/__init__.py b/providers/standard/src/airflow/providers/standard/hooks/__init__.py similarity index 100% rename from providers/tests/alibaba/cloud/log/__init__.py rename to providers/standard/src/airflow/providers/standard/hooks/__init__.py diff --git a/providers/src/airflow/providers/standard/hooks/filesystem.py b/providers/standard/src/airflow/providers/standard/hooks/filesystem.py similarity index 100% rename from providers/src/airflow/providers/standard/hooks/filesystem.py rename to providers/standard/src/airflow/providers/standard/hooks/filesystem.py diff --git a/providers/src/airflow/providers/standard/hooks/package_index.py b/providers/standard/src/airflow/providers/standard/hooks/package_index.py similarity index 100% rename from providers/src/airflow/providers/standard/hooks/package_index.py rename to providers/standard/src/airflow/providers/standard/hooks/package_index.py diff --git a/providers/src/airflow/providers/standard/hooks/subprocess.py b/providers/standard/src/airflow/providers/standard/hooks/subprocess.py similarity index 100% rename from providers/src/airflow/providers/standard/hooks/subprocess.py rename to providers/standard/src/airflow/providers/standard/hooks/subprocess.py diff --git a/providers/tests/alibaba/cloud/operators/__init__.py b/providers/standard/src/airflow/providers/standard/operators/__init__.py similarity index 100% rename from providers/tests/alibaba/cloud/operators/__init__.py rename to providers/standard/src/airflow/providers/standard/operators/__init__.py diff --git a/providers/src/airflow/providers/standard/operators/bash.py b/providers/standard/src/airflow/providers/standard/operators/bash.py similarity index 100% rename from providers/src/airflow/providers/standard/operators/bash.py rename to providers/standard/src/airflow/providers/standard/operators/bash.py diff --git a/providers/src/airflow/providers/standard/operators/datetime.py b/providers/standard/src/airflow/providers/standard/operators/datetime.py similarity index 100% rename from providers/src/airflow/providers/standard/operators/datetime.py rename to providers/standard/src/airflow/providers/standard/operators/datetime.py diff --git a/providers/src/airflow/providers/standard/operators/generic_transfer.py b/providers/standard/src/airflow/providers/standard/operators/generic_transfer.py similarity index 100% rename from providers/src/airflow/providers/standard/operators/generic_transfer.py rename to providers/standard/src/airflow/providers/standard/operators/generic_transfer.py diff --git a/providers/src/airflow/providers/standard/operators/latest_only.py b/providers/standard/src/airflow/providers/standard/operators/latest_only.py similarity index 100% rename from providers/src/airflow/providers/standard/operators/latest_only.py rename to providers/standard/src/airflow/providers/standard/operators/latest_only.py diff --git a/providers/src/airflow/providers/standard/operators/python.py b/providers/standard/src/airflow/providers/standard/operators/python.py similarity index 100% rename from providers/src/airflow/providers/standard/operators/python.py rename to providers/standard/src/airflow/providers/standard/operators/python.py diff --git a/providers/src/airflow/providers/standard/operators/trigger_dagrun.py b/providers/standard/src/airflow/providers/standard/operators/trigger_dagrun.py similarity index 100% rename from providers/src/airflow/providers/standard/operators/trigger_dagrun.py rename to providers/standard/src/airflow/providers/standard/operators/trigger_dagrun.py diff --git a/providers/src/airflow/providers/standard/operators/weekday.py b/providers/standard/src/airflow/providers/standard/operators/weekday.py similarity index 100% rename from providers/src/airflow/providers/standard/operators/weekday.py rename to providers/standard/src/airflow/providers/standard/operators/weekday.py diff --git a/providers/tests/alibaba/cloud/sensors/__init__.py b/providers/standard/src/airflow/providers/standard/sensors/__init__.py similarity index 100% rename from providers/tests/alibaba/cloud/sensors/__init__.py rename to providers/standard/src/airflow/providers/standard/sensors/__init__.py diff --git a/providers/src/airflow/providers/standard/sensors/bash.py b/providers/standard/src/airflow/providers/standard/sensors/bash.py similarity index 100% rename from providers/src/airflow/providers/standard/sensors/bash.py rename to providers/standard/src/airflow/providers/standard/sensors/bash.py diff --git a/providers/src/airflow/providers/standard/sensors/date_time.py b/providers/standard/src/airflow/providers/standard/sensors/date_time.py similarity index 100% rename from providers/src/airflow/providers/standard/sensors/date_time.py rename to providers/standard/src/airflow/providers/standard/sensors/date_time.py diff --git a/providers/src/airflow/providers/standard/sensors/external_task.py b/providers/standard/src/airflow/providers/standard/sensors/external_task.py similarity index 100% rename from providers/src/airflow/providers/standard/sensors/external_task.py rename to providers/standard/src/airflow/providers/standard/sensors/external_task.py diff --git a/providers/src/airflow/providers/standard/sensors/filesystem.py b/providers/standard/src/airflow/providers/standard/sensors/filesystem.py similarity index 100% rename from providers/src/airflow/providers/standard/sensors/filesystem.py rename to providers/standard/src/airflow/providers/standard/sensors/filesystem.py diff --git a/providers/src/airflow/providers/standard/sensors/python.py b/providers/standard/src/airflow/providers/standard/sensors/python.py similarity index 100% rename from providers/src/airflow/providers/standard/sensors/python.py rename to providers/standard/src/airflow/providers/standard/sensors/python.py diff --git a/providers/src/airflow/providers/standard/sensors/time.py b/providers/standard/src/airflow/providers/standard/sensors/time.py similarity index 100% rename from providers/src/airflow/providers/standard/sensors/time.py rename to providers/standard/src/airflow/providers/standard/sensors/time.py diff --git a/providers/src/airflow/providers/standard/sensors/time_delta.py b/providers/standard/src/airflow/providers/standard/sensors/time_delta.py similarity index 100% rename from providers/src/airflow/providers/standard/sensors/time_delta.py rename to providers/standard/src/airflow/providers/standard/sensors/time_delta.py diff --git a/providers/src/airflow/providers/standard/sensors/weekday.py b/providers/standard/src/airflow/providers/standard/sensors/weekday.py similarity index 100% rename from providers/src/airflow/providers/standard/sensors/weekday.py rename to providers/standard/src/airflow/providers/standard/sensors/weekday.py diff --git a/providers/tests/alibaba/cloud/utils/__init__.py b/providers/standard/src/airflow/providers/standard/triggers/__init__.py similarity index 100% rename from providers/tests/alibaba/cloud/utils/__init__.py rename to providers/standard/src/airflow/providers/standard/triggers/__init__.py diff --git a/providers/src/airflow/providers/standard/triggers/external_task.py b/providers/standard/src/airflow/providers/standard/triggers/external_task.py similarity index 100% rename from providers/src/airflow/providers/standard/triggers/external_task.py rename to providers/standard/src/airflow/providers/standard/triggers/external_task.py diff --git a/providers/src/airflow/providers/standard/triggers/file.py b/providers/standard/src/airflow/providers/standard/triggers/file.py similarity index 100% rename from providers/src/airflow/providers/standard/triggers/file.py rename to providers/standard/src/airflow/providers/standard/triggers/file.py diff --git a/providers/src/airflow/providers/standard/triggers/temporal.py b/providers/standard/src/airflow/providers/standard/triggers/temporal.py similarity index 100% rename from providers/src/airflow/providers/standard/triggers/temporal.py rename to providers/standard/src/airflow/providers/standard/triggers/temporal.py diff --git a/providers/tests/standard/__init__.py b/providers/standard/src/airflow/providers/standard/utils/__init__.py similarity index 100% rename from providers/tests/standard/__init__.py rename to providers/standard/src/airflow/providers/standard/utils/__init__.py diff --git a/providers/src/airflow/providers/standard/utils/python_virtualenv.py b/providers/standard/src/airflow/providers/standard/utils/python_virtualenv.py similarity index 100% rename from providers/src/airflow/providers/standard/utils/python_virtualenv.py rename to providers/standard/src/airflow/providers/standard/utils/python_virtualenv.py diff --git a/providers/src/airflow/providers/standard/utils/python_virtualenv_script.jinja2 b/providers/standard/src/airflow/providers/standard/utils/python_virtualenv_script.jinja2 similarity index 100% rename from providers/src/airflow/providers/standard/utils/python_virtualenv_script.jinja2 rename to providers/standard/src/airflow/providers/standard/utils/python_virtualenv_script.jinja2 diff --git a/providers/src/airflow/providers/standard/utils/sensor_helper.py b/providers/standard/src/airflow/providers/standard/utils/sensor_helper.py similarity index 100% rename from providers/src/airflow/providers/standard/utils/sensor_helper.py rename to providers/standard/src/airflow/providers/standard/utils/sensor_helper.py diff --git a/providers/src/airflow/providers/standard/version_compat.py b/providers/standard/src/airflow/providers/standard/version_compat.py similarity index 100% rename from providers/src/airflow/providers/standard/version_compat.py rename to providers/standard/src/airflow/providers/standard/version_compat.py diff --git a/providers/standard/tests/conftest.py b/providers/standard/tests/conftest.py new file mode 100644 index 00000000000000..068fe6bbf5ae9a --- /dev/null +++ b/providers/standard/tests/conftest.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pathlib + +import pytest + +pytest_plugins = "tests_common.pytest_plugin" + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + deprecations_ignore_path = pathlib.Path(__file__).parent.joinpath("deprecations_ignore.yml") + dep_path = [deprecations_ignore_path] if deprecations_ignore_path.exists() else [] + config.inicfg["airflow_deprecations_ignore"] = ( + config.inicfg.get("airflow_deprecations_ignore", []) + dep_path # type: ignore[assignment,operator] + ) diff --git a/providers/standard/tests/provider_tests/__init__.py b/providers/standard/tests/provider_tests/__init__.py new file mode 100644 index 00000000000000..e8fd22856438c4 --- /dev/null +++ b/providers/standard/tests/provider_tests/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/tests/standard/hooks/__init__.py b/providers/standard/tests/provider_tests/standard/__init__.py similarity index 100% rename from providers/tests/standard/hooks/__init__.py rename to providers/standard/tests/provider_tests/standard/__init__.py diff --git a/providers/tests/standard/operators/__init__.py b/providers/standard/tests/provider_tests/standard/decorators/__init__.py similarity index 100% rename from providers/tests/standard/operators/__init__.py rename to providers/standard/tests/provider_tests/standard/decorators/__init__.py diff --git a/tests/decorators/test_python.py b/providers/standard/tests/provider_tests/standard/decorators/test_python.py similarity index 99% rename from tests/decorators/test_python.py rename to providers/standard/tests/provider_tests/standard/decorators/test_python.py index bbaa4236f3100c..f904d10fc340f1 100644 --- a/tests/decorators/test_python.py +++ b/providers/standard/tests/provider_tests/standard/decorators/test_python.py @@ -27,19 +27,19 @@ from airflow.decorators.base import DecoratedMappedOperator from airflow.exceptions import AirflowException, XComNotFound from airflow.models.expandinput import DictOfListsExpandInput +from airflow.models.mappedoperator import MappedOperator from airflow.models.taskinstance import TaskInstance from airflow.models.taskmap import TaskMap from airflow.models.xcom_arg import PlainXComArg from airflow.sdk import DAG, BaseOperator, TaskGroup, XComArg -from airflow.sdk.definitions.mappedoperator import MappedOperator from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType from airflow.utils.xcom import XCOM_RETURN_KEY +from provider_tests.standard.operators.test_python import BasePythonTest -from providers.tests.standard.operators.test_python import BasePythonTest from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: diff --git a/providers/tests/standard/sensors/__init__.py b/providers/standard/tests/provider_tests/standard/hooks/__init__.py similarity index 100% rename from providers/tests/standard/sensors/__init__.py rename to providers/standard/tests/provider_tests/standard/hooks/__init__.py diff --git a/providers/tests/standard/hooks/test_filesystem.py b/providers/standard/tests/provider_tests/standard/hooks/test_filesystem.py similarity index 100% rename from providers/tests/standard/hooks/test_filesystem.py rename to providers/standard/tests/provider_tests/standard/hooks/test_filesystem.py diff --git a/providers/tests/standard/hooks/test_package_index.py b/providers/standard/tests/provider_tests/standard/hooks/test_package_index.py similarity index 100% rename from providers/tests/standard/hooks/test_package_index.py rename to providers/standard/tests/provider_tests/standard/hooks/test_package_index.py diff --git a/providers/tests/standard/hooks/test_subprocess.py b/providers/standard/tests/provider_tests/standard/hooks/test_subprocess.py similarity index 100% rename from providers/tests/standard/hooks/test_subprocess.py rename to providers/standard/tests/provider_tests/standard/hooks/test_subprocess.py diff --git a/providers/tests/standard/triggers/__init__.py b/providers/standard/tests/provider_tests/standard/operators/__init__.py similarity index 100% rename from providers/tests/standard/triggers/__init__.py rename to providers/standard/tests/provider_tests/standard/operators/__init__.py diff --git a/providers/tests/standard/operators/test_bash.py b/providers/standard/tests/provider_tests/standard/operators/test_bash.py similarity index 100% rename from providers/tests/standard/operators/test_bash.py rename to providers/standard/tests/provider_tests/standard/operators/test_bash.py diff --git a/providers/tests/standard/operators/test_datetime.py b/providers/standard/tests/provider_tests/standard/operators/test_datetime.py similarity index 100% rename from providers/tests/standard/operators/test_datetime.py rename to providers/standard/tests/provider_tests/standard/operators/test_datetime.py diff --git a/providers/tests/standard/operators/test_generic_transfer.py b/providers/standard/tests/provider_tests/standard/operators/test_generic_transfer.py similarity index 100% rename from providers/tests/standard/operators/test_generic_transfer.py rename to providers/standard/tests/provider_tests/standard/operators/test_generic_transfer.py diff --git a/providers/tests/standard/operators/test_latest_only_operator.py b/providers/standard/tests/provider_tests/standard/operators/test_latest_only_operator.py similarity index 100% rename from providers/tests/standard/operators/test_latest_only_operator.py rename to providers/standard/tests/provider_tests/standard/operators/test_latest_only_operator.py diff --git a/providers/tests/standard/operators/test_python.py b/providers/standard/tests/provider_tests/standard/operators/test_python.py similarity index 100% rename from providers/tests/standard/operators/test_python.py rename to providers/standard/tests/provider_tests/standard/operators/test_python.py diff --git a/providers/tests/standard/operators/test_weekday.py b/providers/standard/tests/provider_tests/standard/operators/test_weekday.py similarity index 100% rename from providers/tests/standard/operators/test_weekday.py rename to providers/standard/tests/provider_tests/standard/operators/test_weekday.py diff --git a/providers/tests/standard/utils/__init__.py b/providers/standard/tests/provider_tests/standard/sensors/__init__.py similarity index 100% rename from providers/tests/standard/utils/__init__.py rename to providers/standard/tests/provider_tests/standard/sensors/__init__.py diff --git a/providers/tests/standard/sensors/test_bash.py b/providers/standard/tests/provider_tests/standard/sensors/test_bash.py similarity index 100% rename from providers/tests/standard/sensors/test_bash.py rename to providers/standard/tests/provider_tests/standard/sensors/test_bash.py diff --git a/providers/tests/standard/sensors/test_date_time.py b/providers/standard/tests/provider_tests/standard/sensors/test_date_time.py similarity index 100% rename from providers/tests/standard/sensors/test_date_time.py rename to providers/standard/tests/provider_tests/standard/sensors/test_date_time.py diff --git a/providers/tests/standard/sensors/test_python.py b/providers/standard/tests/provider_tests/standard/sensors/test_python.py similarity index 97% rename from providers/tests/standard/sensors/test_python.py rename to providers/standard/tests/provider_tests/standard/sensors/test_python.py index 2d5dc570031d8b..4f11a1da46a5f3 100644 --- a/providers/tests/standard/sensors/test_python.py +++ b/providers/standard/tests/provider_tests/standard/sensors/test_python.py @@ -25,8 +25,7 @@ from airflow.exceptions import AirflowSensorTimeout from airflow.providers.standard.sensors.python import PythonSensor from airflow.sensors.base import PokeReturnValue - -from providers.tests.standard.operators.test_python import BasePythonTest +from provider_tests.standard.operators.test_python import BasePythonTest pytestmark = pytest.mark.db_test diff --git a/providers/tests/standard/sensors/test_time.py b/providers/standard/tests/provider_tests/standard/sensors/test_time.py similarity index 100% rename from providers/tests/standard/sensors/test_time.py rename to providers/standard/tests/provider_tests/standard/sensors/test_time.py diff --git a/providers/tests/standard/sensors/test_time_delta.py b/providers/standard/tests/provider_tests/standard/sensors/test_time_delta.py similarity index 100% rename from providers/tests/standard/sensors/test_time_delta.py rename to providers/standard/tests/provider_tests/standard/sensors/test_time_delta.py diff --git a/providers/tests/standard/sensors/test_weekday.py b/providers/standard/tests/provider_tests/standard/sensors/test_weekday.py similarity index 100% rename from providers/tests/standard/sensors/test_weekday.py rename to providers/standard/tests/provider_tests/standard/sensors/test_weekday.py diff --git a/providers/tests/system/alibaba/__init__.py b/providers/standard/tests/provider_tests/standard/triggers/__init__.py similarity index 100% rename from providers/tests/system/alibaba/__init__.py rename to providers/standard/tests/provider_tests/standard/triggers/__init__.py diff --git a/providers/tests/standard/triggers/test_external_task.py b/providers/standard/tests/provider_tests/standard/triggers/test_external_task.py similarity index 100% rename from providers/tests/standard/triggers/test_external_task.py rename to providers/standard/tests/provider_tests/standard/triggers/test_external_task.py diff --git a/providers/tests/standard/triggers/test_file.py b/providers/standard/tests/provider_tests/standard/triggers/test_file.py similarity index 100% rename from providers/tests/standard/triggers/test_file.py rename to providers/standard/tests/provider_tests/standard/triggers/test_file.py diff --git a/providers/tests/standard/triggers/test_temporal.py b/providers/standard/tests/provider_tests/standard/triggers/test_temporal.py similarity index 100% rename from providers/tests/standard/triggers/test_temporal.py rename to providers/standard/tests/provider_tests/standard/triggers/test_temporal.py diff --git a/providers/standard/tests/provider_tests/standard/utils/__init__.py b/providers/standard/tests/provider_tests/standard/utils/__init__.py new file mode 100644 index 00000000000000..13a83393a9124b --- /dev/null +++ b/providers/standard/tests/provider_tests/standard/utils/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/tests/standard/utils/test_python_virtualenv.py b/providers/standard/tests/provider_tests/standard/utils/test_python_virtualenv.py similarity index 100% rename from providers/tests/standard/utils/test_python_virtualenv.py rename to providers/standard/tests/provider_tests/standard/utils/test_python_virtualenv.py diff --git a/pyproject.toml b/pyproject.toml index fdb8a138666ba7..48b8ba57324dfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -322,12 +322,18 @@ testing = ["dev", "providers.tests", "task_sdk.tests", "tests_common", "tests"] "airflow/__init__.py" = ["F401", "TC004", "I002"] "airflow/models/__init__.py" = ["F401", "TC004"] "airflow/models/sqla_models.py" = ["F401"] + +# Those are needed so that __init__.py chaining of packages properly works for IDEs +# the first non-comment line of such empty __init__.py files should be: +# __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore "providers/src/airflow/providers/__init__.py" = ["I002"] "providers/src/airflow/__init__.py" = ["I002"] +"providers/*/tests/*/__init__.py" = ["I002"] +"providers/*/*/tests/*/*/__init__.py" = ["I002"] # The test_python.py is needed because adding __future__.annotations breaks runtime checks that are # needed for the test to work -"tests/decorators/test_python.py" = ["I002"] +"providers/standard/tests/provider_tests/standard/decorators/test_python.py" = ["I002"] # The Pydantic representations of SqlAlchemy Models are not parsed well with Pydantic # when __future__.annotations is used so we need to skip them from upgrading @@ -606,29 +612,35 @@ ignore_errors = true dev = [ # TODO(potiuk): remove me when all providers are moved to new structure "local-providers", - "apache-airflow-task-sdk", "apache-airflow-providers-airbyte", + "apache-airflow-providers-alibaba", "apache-airflow-providers-apache-iceberg", "apache-airflow-providers-celery", "apache-airflow-providers-edge", + "apache-airflow-providers-standard", + "apache-airflow-task-sdk", ] [tool.uv.sources] # These names must match the names as defined in the pyproject.toml of the workspace items, # *not* the workspace folder paths local-providers = { workspace = true } -apache-airflow-task-sdk = { workspace = true } apache-airflow-providers-airbyte = {workspace = true} +apache-airflow-providers-alibaba = { workspace = true } apache-airflow-providers-apache-iceberg = {workspace = true} apache-airflow-providers-celery = {workspace = true} apache-airflow-providers-edge = {workspace = true} +apache-airflow-providers-standard = { workspace = true } +apache-airflow-task-sdk = { workspace = true } [tool.uv.workspace] members = [ "providers", - "task_sdk", "providers/airbyte", + "providers/alibaba", "providers/apache/iceberg", "providers/celery", - "providers/edge" + "providers/edge", + "providers/standard", + "task_sdk", ] diff --git a/scripts/in_container/run_fix_ownership.py b/scripts/in_container/run_fix_ownership.py index bb25fc17bdc44f..184c6da0bb9d24 100755 --- a/scripts/in_container/run_fix_ownership.py +++ b/scripts/in_container/run_fix_ownership.py @@ -44,7 +44,7 @@ def change_ownership_of_files(path: Path) -> None: sys.exit(1) count_files = 0 root_uid = pwd.getpwnam("root").pw_uid - skip_folders = {".venv", "node_modules", ".mypy_cache", ".pytest_cache", ".ruff_cache"} + skip_folders = {".venv", "node_modules"} for root, dirs, files in os.walk(path): original_length = len(dirs) original_dirs = dirs.copy() @@ -57,6 +57,8 @@ def change_ownership_of_files(path: Path) -> None: f"{root}: Skipped {original_length - new_length} " f"folders: {set(original_dirs) - set(dirs)}" ) + for dir_name in dirs: + os.chown(Path(root) / dir_name, int(host_user_id), int(host_group_id), follow_symlinks=False) for name in files: file = Path(root) / name try: diff --git a/tests_common/test_utils/__init__.py b/tests_common/test_utils/__init__.py index c27a6b55209c15..336a6ca747790d 100644 --- a/tests_common/test_utils/__init__.py +++ b/tests_common/test_utils/__init__.py @@ -19,4 +19,4 @@ from pathlib import Path -AIRFLOW_MAIN_FOLDER = Path(__file__).parents[2] +AIRFLOW_MAIN_FOLDER = Path(__file__).resolve().parents[2]