From b133d20fa7020b66786f15a9c37423dd2baa8d71 Mon Sep 17 00:00:00 2001 From: Chris Okorodudu Date: Mon, 13 May 2024 10:41:06 -0400 Subject: [PATCH 1/4] alter update_node_dependency method to include relevant test nodes --- cosmos/dbt/graph.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 17987e327..b23be6ba1 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -440,8 +440,9 @@ def update_node_dependency(self) -> None: Updates in-place: * self.filtered_nodes """ - for _, node in self.filtered_nodes.items(): + for _, node in self.nodes.items(): if node.resource_type == DbtResourceType.TEST: for node_id in node.depends_on: if node_id in self.filtered_nodes: self.filtered_nodes[node_id].has_test = True + self.filtered_nodes[node.unique_id] = node From b5fc7007a23f805e6b973fbf818d5d497c00d8b3 Mon Sep 17 00:00:00 2001 From: Chris Okorodudu Date: Mon, 13 May 2024 11:05:07 -0400 Subject: [PATCH 2/4] add test --- tests/dbt/test_graph.py | 71 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 22cd5c617..5f778b763 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -110,6 +110,77 @@ def test_load_via_manifest_with_exclude(project_name, manifest_filepath, model_f assert sample_node.file_path == DBT_PROJECTS_ROOT_DIR / f"{project_name}/models/{model_filepath}" +@pytest.mark.parametrize( + "project_name,manifest_filepath,model_filepath", + [(DBT_PROJECT_NAME, SAMPLE_MANIFEST, "customers.sql"), ("jaffle_shop_python", SAMPLE_MANIFEST_PY, "customers.py")], +) +def test_load_via_manifest_with_select(project_name, manifest_filepath, model_filepath): + project_config = ProjectConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, manifest_path=manifest_filepath + ) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + render_config = RenderConfig(select=["+customers"]) + execution_config = ExecutionConfig(dbt_project_path=project_config.dbt_project_path) + dbt_graph = DbtGraph( + project=project_config, + execution_config=execution_config, + profile_config=profile_config, + render_config=render_config, + ) + dbt_graph.load_from_dbt_manifest() + + expected_keys = [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders", + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments", + "seed.jaffle_shop.raw_customers", + "seed.jaffle_shop.raw_orders", + "seed.jaffle_shop.raw_payments", + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3", + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", + "test.jaffle_shop.not_null_orders_amount.106140f9fd", + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1", + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e", + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada", + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a", + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712", + ] + assert sorted(dbt_graph.nodes.keys()) == expected_keys + + assert len(dbt_graph.nodes) == 28 + assert len(dbt_graph.filtered_nodes) == 7 + assert "model.jaffle_shop.orders" not in dbt_graph.filtered_nodes + + sample_node = dbt_graph.nodes["model.jaffle_shop.customers"] + assert sample_node.name == "customers" + assert sample_node.unique_id == "model.jaffle_shop.customers" + assert sample_node.resource_type == DbtResourceType.MODEL + assert sample_node.depends_on == [ + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments", + ] + assert sample_node.file_path == DBT_PROJECTS_ROOT_DIR / f"{project_name}/models/{model_filepath}" + + @patch("cosmos.dbt.graph.DbtGraph.load_from_dbt_manifest", return_value=None) def test_load_automatic_manifest_is_available(mock_load_from_dbt_manifest): project_config = ProjectConfig( From b065bd3e130b9f6c2b450b42a5bf894681806bce Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Tue, 14 May 2024 17:40:20 +0530 Subject: [PATCH 3/4] Enable pre-commit run and fix type-check job (#957) closes: https://github.com/astronomer/astronomer-cosmos/issues/952 This PR addresses the following issues: **1. Pre-commit Setup:** - Pre-commit was not running in the CI pipeline because it was neither configured for this repository nor were we using the pre-commit command in CI to run it manually. To fix this: Installed and added pre-commit to this repository. This allows pre-commit to automatically run the pre-commit configuration on every push. **2. Type Check Job:** - Previously, the type-check job was running with mypy directly, and the pre-commit type check command had different configurations. This PR aligns the configurations. **3. Type Check Fixes:** - Ignored the type check for __dataclass_fields__ since mypy seems to not recognize it. - Added a check for None for ti.task --- cosmos/operators/local.py | 2 ++ cosmos/profiles/base.py | 3 ++- pyproject.toml | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cosmos/operators/local.py b/cosmos/operators/local.py index 390c10244..a9b95baf7 100644 --- a/cosmos/operators/local.py +++ b/cosmos/operators/local.py @@ -238,6 +238,8 @@ def store_compiled_sql(self, tmp_project_dir: str, context: Context, session: Se ti = context["ti"] if isinstance(ti, TaskInstance): # verifies ti is a TaskInstance in order to access and use the "task" field + if TYPE_CHECKING: + assert ti.task is not None ti.task.template_fields = self.template_fields rtif = RenderedTaskInstanceFields(ti, render_templates=False) diff --git a/cosmos/profiles/base.py b/cosmos/profiles/base.py index 308d71a81..d4b44b591 100755 --- a/cosmos/profiles/base.py +++ b/cosmos/profiles/base.py @@ -42,7 +42,8 @@ class DbtProfileConfigVars: def as_dict(self) -> dict[str, Any] | None: result = { field.name: getattr(self, field.name) - for field in self.__dataclass_fields__.values() + # Look like the __dataclass_fields__ attribute is not recognized by mypy + for field in self.__dataclass_fields__.values() # type: ignore[attr-defined] if getattr(self, field.name) is not None } if result != {}: diff --git a/pyproject.toml b/pyproject.toml index f740f2071..97788aee5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,8 +78,8 @@ tests = [ "pytest-describe", "sqlalchemy-stubs", # Change when sqlalchemy is upgraded https://docs.sqlalchemy.org/en/14/orm/extensions/mypy.html "types-requests", - "mypy", "sqlalchemy-stubs", # Change when sqlalchemy is upgraded https://docs.sqlalchemy.org/en/14/orm/extensions/mypy.html + "pre-commit", ] docker = [ "apache-airflow-providers-docker>=3.5.0", @@ -152,7 +152,7 @@ test-integration-sqlite = 'sh scripts/test/integration-sqlite.sh' test-integration-sqlite-setup = 'sh scripts/test/integration-sqlite-setup.sh' test-performance = 'sh scripts/test/performance.sh' test-performance-setup = 'sh scripts/test/performance-setup.sh' -type-check = "mypy cosmos" +type-check = " pre-commit run mypy --files cosmos/**/*" [tool.pytest.ini_options] filterwarnings = ["ignore::DeprecationWarning"] From ff26b39393bc59901fa53d96187cf3273c357c7c Mon Sep 17 00:00:00 2001 From: Chris Okorodudu Date: Tue, 14 May 2024 21:33:38 -0400 Subject: [PATCH 4/4] fix for test failure --- cosmos/dbt/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index b23be6ba1..c37932caf 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -440,7 +440,7 @@ def update_node_dependency(self) -> None: Updates in-place: * self.filtered_nodes """ - for _, node in self.nodes.items(): + for _, node in list(self.nodes.items()): if node.resource_type == DbtResourceType.TEST: for node_id in node.depends_on: if node_id in self.filtered_nodes: