From 8394209559f432e197dbc3d2067437f7455a3461 Mon Sep 17 00:00:00 2001 From: Roman Korsun Date: Thu, 16 Nov 2023 00:29:52 +0200 Subject: [PATCH 1/8] feat: add detailed_table_type param for relation output in list_relations_without_caching method --- dbt/adapters/athena/impl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/adapters/athena/impl.py b/dbt/adapters/athena/impl.py index a74326a1..f2f4da11 100755 --- a/dbt/adapters/athena/impl.py +++ b/dbt/adapters/athena/impl.py @@ -724,6 +724,7 @@ def list_relations_without_caching(self, schema_relation: AthenaRelation) -> Lis identifier=table["Name"], quote_policy=quote_policy, type=_type, + detailed_table_type=table["Parameters"]["table_type"], ) ) except ClientError as e: From 332d55b8fcf9f2bcf26dca13a823c545894a0445 Mon Sep 17 00:00:00 2001 From: Roman Korsun Date: Thu, 16 Nov 2023 00:52:36 +0200 Subject: [PATCH 2/8] fix: test gpg sign --- dbt/adapters/athena/impl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/adapters/athena/impl.py b/dbt/adapters/athena/impl.py index f2f4da11..f2a0e9ec 100755 --- a/dbt/adapters/athena/impl.py +++ b/dbt/adapters/athena/impl.py @@ -725,6 +725,7 @@ def list_relations_without_caching(self, schema_relation: AthenaRelation) -> Lis quote_policy=quote_policy, type=_type, detailed_table_type=table["Parameters"]["table_type"], + test=table["test"]["test"], ) ) except ClientError as e: From 624e8b8b494e6562003f6be3740d8d2c2bad71f0 Mon Sep 17 00:00:00 2001 From: Roman Korsun Date: Thu, 16 Nov 2023 00:53:54 +0200 Subject: [PATCH 3/8] fix: gpg sign fixed --- dbt/adapters/athena/impl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/athena/impl.py b/dbt/adapters/athena/impl.py index f2a0e9ec..f2f4da11 100755 --- a/dbt/adapters/athena/impl.py +++ b/dbt/adapters/athena/impl.py @@ -725,7 +725,6 @@ def list_relations_without_caching(self, schema_relation: AthenaRelation) -> Lis quote_policy=quote_policy, type=_type, detailed_table_type=table["Parameters"]["table_type"], - test=table["test"]["test"], ) ) except ClientError as e: From 0cf01ff446d657c0269b3d458ab311a82564508e Mon Sep 17 00:00:00 2001 From: Roman Korsun Date: Fri, 17 Nov 2023 16:48:38 +0200 Subject: [PATCH 4/8] feat: add detailed_table_type param in AthenaRelation class --- dbt/adapters/athena/relation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/adapters/athena/relation.py b/dbt/adapters/athena/relation.py index 680f9e09..ca1bb284 100644 --- a/dbt/adapters/athena/relation.py +++ b/dbt/adapters/athena/relation.py @@ -31,6 +31,7 @@ class AthenaRelation(BaseRelation): quote_character: str = '"' # Presto quote character include_policy: Policy = field(default_factory=lambda: AthenaIncludePolicy()) s3_path_table_part: Optional[str] = None + detailed_table_type: Optional[str] = None # table_type option from the table Parameters in Glue Catalog def render_hive(self) -> str: """ From 68e4d86f0a10b20d7c484dccdffd4143d96b274c Mon Sep 17 00:00:00 2001 From: Roman Korsun Date: Fri, 17 Nov 2023 17:27:40 +0200 Subject: [PATCH 5/8] fix: Fix KeyError in table type retrieval for AthenaRelation --- dbt/adapters/athena/impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/athena/impl.py b/dbt/adapters/athena/impl.py index f2f4da11..cda262be 100755 --- a/dbt/adapters/athena/impl.py +++ b/dbt/adapters/athena/impl.py @@ -712,6 +712,7 @@ def list_relations_without_caching(self, schema_relation: AthenaRelation) -> Lis LOGGER.debug(f"Table '{table['Name']}' has no TableType attribute - Ignoring") continue _type = table["TableType"] + _detailed_table_type = table["Parameters"].get("table_type", None) if _type == "VIRTUAL_VIEW": _type = self.Relation.View else: @@ -724,7 +725,7 @@ def list_relations_without_caching(self, schema_relation: AthenaRelation) -> Lis identifier=table["Name"], quote_policy=quote_policy, type=_type, - detailed_table_type=table["Parameters"]["table_type"], + detailed_table_type=_detailed_table_type, ) ) except ClientError as e: From d074967317cf12d191f5f5b4f58b0cedf51f7f5c Mon Sep 17 00:00:00 2001 From: Roman Korsun Date: Sat, 18 Nov 2023 23:37:34 +0200 Subject: [PATCH 6/8] Fix KeyError in unittests, add iceberg table in test_list_relations_without_caching, add assertions for detailed_table_type parameter --- dbt/adapters/athena/impl.py | 2 +- tests/unit/test_adapter.py | 17 +++++++++++++---- tests/unit/utils.py | 8 +++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/dbt/adapters/athena/impl.py b/dbt/adapters/athena/impl.py index cda262be..ec1bf994 100755 --- a/dbt/adapters/athena/impl.py +++ b/dbt/adapters/athena/impl.py @@ -712,7 +712,7 @@ def list_relations_without_caching(self, schema_relation: AthenaRelation) -> Lis LOGGER.debug(f"Table '{table['Name']}' has no TableType attribute - Ignoring") continue _type = table["TableType"] - _detailed_table_type = table["Parameters"].get("table_type", None) + _detailed_table_type = table["Parameters"].get("table_type", "") if _type == "VIRTUAL_VIEW": _type = self.Relation.View else: diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 22a4d24f..d1cf69e5 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -857,18 +857,25 @@ def test__get_data_catalog(self, mock_aws_service): def _test_list_relations_without_caching(self, schema_relation): self.adapter.acquire_connection("dummy") relations = self.adapter.list_relations_without_caching(schema_relation) - assert len(relations) == 3 + assert len(relations) == 4 assert all(isinstance(rel, AthenaRelation) for rel in relations) relations.sort(key=lambda rel: rel.name) - other = relations[0] - table = relations[1] - view = relations[2] + iceberg_table = relations[0] + other = relations[1] + table = relations[2] + view = relations[3] + assert iceberg_table.name == "iceberg" + assert iceberg_table.type == "table" + assert iceberg_table.detailed_table_type == "ICEBERG" assert other.name == "other" assert other.type == "table" + assert other.detailed_table_type == "" assert table.name == "table" assert table.type == "table" + assert table.detailed_table_type == "" assert view.name == "view" assert view.type == "view" + assert view.detailed_table_type == "" @mock_athena @mock_glue @@ -880,6 +887,7 @@ def test_list_relations_without_caching_with_awsdatacatalog(self, mock_aws_servi mock_aws_service.create_table("other") mock_aws_service.create_view("view") mock_aws_service.create_table_without_table_type("without_table_type") + mock_aws_service.create_iceberg_table("iceberg") schema_relation = self.adapter.Relation.create( database=DATA_CATALOG_NAME, schema=DATABASE_NAME, @@ -897,6 +905,7 @@ def test_list_relations_without_caching_with_other_glue_data_catalog(self, mock_ mock_aws_service.create_table("other") mock_aws_service.create_view("view") mock_aws_service.create_table_without_table_type("without_table_type") + mock_aws_service.create_iceberg_table("iceberg") schema_relation = self.adapter.Relation.create( database=data_catalog_name, schema=DATABASE_NAME, diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 097f6ce9..791181cf 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -188,6 +188,9 @@ def create_view(self, view_name: str): "Location": "", }, "TableType": "VIRTUAL_VIEW", + "Parameters": { + "TableOwner": "John Doe", + }, }, ) @@ -329,7 +332,7 @@ def create_iceberg_table(self, table_name: str): "TableType": "EXTERNAL_TABLE", "Parameters": { "metadata_location": f"s3://{BUCKET}/tables/metadata/{table_name}/123.json", - "table_type": "iceberg", + "table_type": "ICEBERG", }, }, ) @@ -349,6 +352,9 @@ def create_table_without_table_type(self, table_name: str): ], "Location": f"s3://{BUCKET}/tables/{table_name}", }, + "Parameters": { + "TableOwner": "John Doe", + }, }, ) From 4d890c4bded7ef00c738a3e73969bbf48d718acf Mon Sep 17 00:00:00 2001 From: Roman Korsun Date: Sun, 19 Nov 2023 17:24:23 +0200 Subject: [PATCH 7/8] Add fucntional test for detailed_table_type parameter in list_relations_without_caching function --- .../adapter/test_detailed_table_type.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/functional/adapter/test_detailed_table_type.py diff --git a/tests/functional/adapter/test_detailed_table_type.py b/tests/functional/adapter/test_detailed_table_type.py new file mode 100644 index 00000000..d6215c08 --- /dev/null +++ b/tests/functional/adapter/test_detailed_table_type.py @@ -0,0 +1,46 @@ +import re + +import pytest + +from dbt.tests.util import run_dbt, run_dbt_and_capture + +get_detailed_table_type_sql = """ +{% macro get_detailed_table_type(schema) %} + {% if execute %} + {% set relation = api.Relation.create(database="awsdatacatalog", schema=schema) %} + {% set schema_tables = adapter.list_relations_without_caching(schema_relation = relation) %} + {{ log(schema_tables) }} + {% for rel in schema_tables %} + {% do log('Detailed Table Type: ' ~ rel.detailed_table_type, info=True) %} + {% endfor %} + {% endif %} +{% endmacro %} +""" + +# Model SQL for an Iceberg table +iceberg_model_sql = """ + select 1 as id, 'iceberg' as name + {{ config(materialized='table', schema='default', table_type='iceberg') }} +""" + + +@pytest.mark.usefixtures("project") +class TestDetailedTableType: + @pytest.fixture(scope="class") + def macros(self): + return {"get_detailed_table_type.sql": get_detailed_table_type_sql} + + @pytest.fixture(scope="class") + def models(self): + return {"iceberg_model.sql": iceberg_model_sql} + + def test_detailed_table_type(self, project): + # Run the models + run_results = run_dbt(["run"]) + assert len(run_results) == 1 # Ensure model ran successfully + + iceberg_schema = run_results.results[0].node.schema + args_str = f'{{"schema": "{iceberg_schema}"}}' + run_macro, stdout = run_dbt_and_capture(["run-operation", "get_detailed_table_type", "--args", args_str]) + iceberg_table_type = re.search(r"Detailed Table Type: (\w+)", stdout).group(1) + assert iceberg_table_type == "ICEBERG" From f02445374ae06a4d6236940c54434aeed54c4a19 Mon Sep 17 00:00:00 2001 From: Roman Korsun Date: Mon, 20 Nov 2023 00:33:02 +0200 Subject: [PATCH 8/8] Minor refactoring of functional test --- tests/functional/adapter/test_detailed_table_type.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/functional/adapter/test_detailed_table_type.py b/tests/functional/adapter/test_detailed_table_type.py index d6215c08..ba6c885c 100644 --- a/tests/functional/adapter/test_detailed_table_type.py +++ b/tests/functional/adapter/test_detailed_table_type.py @@ -9,7 +9,6 @@ {% if execute %} {% set relation = api.Relation.create(database="awsdatacatalog", schema=schema) %} {% set schema_tables = adapter.list_relations_without_caching(schema_relation = relation) %} - {{ log(schema_tables) }} {% for rel in schema_tables %} {% do log('Detailed Table Type: ' ~ rel.detailed_table_type, info=True) %} {% endfor %} @@ -20,7 +19,7 @@ # Model SQL for an Iceberg table iceberg_model_sql = """ select 1 as id, 'iceberg' as name - {{ config(materialized='table', schema='default', table_type='iceberg') }} + {{ config(materialized='table', table_type='iceberg') }} """ @@ -39,8 +38,7 @@ def test_detailed_table_type(self, project): run_results = run_dbt(["run"]) assert len(run_results) == 1 # Ensure model ran successfully - iceberg_schema = run_results.results[0].node.schema - args_str = f'{{"schema": "{iceberg_schema}"}}' + args_str = f'{{"schema": "{project.test_schema}"}}' run_macro, stdout = run_dbt_and_capture(["run-operation", "get_detailed_table_type", "--args", args_str]) iceberg_table_type = re.search(r"Detailed Table Type: (\w+)", stdout).group(1) assert iceberg_table_type == "ICEBERG"