Add pre-commit to check consistency of new provider's workspace setup (…

…#45800)

The new-structure providers require to be added to pyproject.toml
in a few places. This pre-commit makes sure tha this is done.

.pre-commit-config.yaml

@@ -930,6 +930,14 @@ repos:
         files: ^dev/breeze/pyproject\.toml$|^dev/breeze/README\.md$
         pass_filenames: false
         require_serial: true
+      - id: check-pyproject-toml-consistency
+        name: Check consistency of Airflow's pyproject.toml
+        language: python
+        entry: ./scripts/ci/pre_commit/check_pyproject_toml_consistency.py
+        files: ^pyproject.toml$
+        pass_filenames: false
+        require_serial: true
+        additional_dependencies: ['rich>=12.4.4', 'tomli']
       - id: update-reproducible-source-date-epoch
         name: Update Source Date Epoch for reproducible builds
         language: python

contributing-docs/08_static_code_checks.rst

@@ -220,6 +220,8 @@ require Breeze Docker image to be built locally.
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | check-pydevd-left-in-code                                 | Check for pydevd debug statements accidentally left    |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
+| check-pyproject-toml-consistency                          | Check consistency of Airflow's pyproject.toml          |         |
++-----------------------------------------------------------+--------------------------------------------------------+---------+
 | check-revision-heads-map                                  | Check that the REVISION_HEADS_MAP is up-to-date        |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | check-safe-filter-usage-in-html                           | Don't use safe in templates                            |         |

dev/breeze/doc/images/output_static-checks.txt

@@ -1 +1 @@
-c60e6ba1483da43b8708603bc2a3dc9c
+075f067748cbe2f9161815fc5a3f63f8

dev/breeze/src/airflow_breeze/pre_commit_ids.py

@@ -75,6 +75,7 @@
     "check-provider-yaml-valid",
     "check-providers-subpackages-init-file-exist",
     "check-pydevd-left-in-code",
+    "check-pyproject-toml-consistency",
     "check-revision-heads-map",
     "check-safe-filter-usage-in-html",
     "check-sql-dependency-common-data-structure",

scripts/ci/pre_commit/check_pyproject_toml_consistency.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Test for an order of dependencies in setup.py
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.resolve()))  # make sure common_precommit_utils is imported
+from common_precommit_utils import console, get_all_new_provider_ids
+
+AIRFLOW_ROOT_PATH = Path(__file__).parents[3].resolve()
+PYPROJECT_TOML_FILE = AIRFLOW_ROOT_PATH / "pyproject.toml"
+PROVIDERS_DIR = AIRFLOW_ROOT_PATH / "providers"
+
+if __name__ == "__main__":
+    try:
+        import tomllib
+    except ImportError:
+        import tomli as tomllib
+
+    error = False
+    toml_dict = tomllib.loads(PYPROJECT_TOML_FILE.read_text())
+    # TODO(potiuk): renamme when all providers are switched to new style
+    all_new_providers = get_all_new_provider_ids()
+    for provider_id in all_new_providers:
+        expected_provider_package = f"apache-airflow-providers-{provider_id.replace('.', '-')}"
+        expected_member = "providers/" + provider_id.replace(".", "/")
+        dev_dependency_group = toml_dict["dependency-groups"]["dev"]
+        if expected_provider_package not in dev_dependency_group:
+            console.print(
+                f"[red]ERROR: {expected_provider_package} is not found in airflow's pyproject.toml "
+                f"in dev dependency-group: {dev_dependency_group}[/red]"
+            )
+            error = True
+        tool_uv_sources = toml_dict["tool"]["uv"]["sources"]
+        if expected_provider_package not in tool_uv_sources:
+            console.print(
+                f"[red]ERROR: {expected_provider_package} is not found in airflow's pyproject.toml "
+                f"in tool.uv.sources: {tool_uv_sources}[/red]"
+            )
+            error = True
+        tool_uv_workspace_members = toml_dict["tool"]["uv"]["workspace"]["members"]
+        if expected_member not in tool_uv_workspace_members:
+            console.print(
+                f"[red]ERROR: {expected_member} is not found in airflow's pyproject.toml "
+                f"in tool.uv.workspace members: {tool_uv_workspace_members}[/red]"
+            )
+            error = True
+    if error:
+        sys.exit(1)

scripts/ci/pre_commit/common_precommit_utils.py

@@ -30,6 +30,8 @@
 
 AIRFLOW_SOURCES_ROOT_PATH = Path(__file__).parents[3].resolve()
 AIRFLOW_BREEZE_SOURCES_PATH = AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze"
+AIRFLOW_PROVIDERS_ROOT_PATH = AIRFLOW_SOURCES_ROOT_PATH / "providers"
+
 DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.9"
 
 console = Console(width=400, color_system="standard")
@@ -259,3 +261,18 @@ def get_provider_base_dir_from_path(file_path: Path) -> Path | None:
         if (parent / "provider.yaml").exists():
             return parent
     return None
+
+
+# TODO(potiuk): rename this function when all providers are moved to new structure
+def get_all_new_provider_ids() -> list[str]:
+    """
+    Get all providers from the new provider structure
+    """
+    all_provider_ids = []
+    for provider_file in AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml"):
+        if provider_file.is_relative_to(AIRFLOW_PROVIDERS_ROOT_PATH / "src"):
+            continue
+        provider_id = get_provider_id_from_path(provider_file)
+        if provider_id:
+            all_provider_ids.append(provider_id)
+    return all_provider_ids