From f638a3d50c62c386183b9b074ff27476600cfbcc Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Fri, 16 Oct 2020 18:38:22 +0200 Subject: [PATCH 01/22] Store relation name in manifest's node object --- core/dbt/compilation.py | 7 +++++++ core/dbt/contracts/graph/compiled.py | 1 + test/unit/test_manifest.py | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 5811227efe4..cf1124e7d52 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -169,6 +169,11 @@ def add_ephemeral_prefix(self, name: str): relation_cls = adapter.Relation return relation_cls.add_ephemeral_prefix(name) + def _get_relation_name(self, node: ParsedNode): + adapter = get_adapter(self.config) + relation_cls = adapter.Relation + return str(relation_cls.create_from(self.config, node)) + def _get_compiled_model( self, manifest: Manifest, @@ -388,6 +393,8 @@ def _compile_node( node, ) + compiled_node.relation_name = self._get_relation_name(node) + compiled_node.compiled = True injected_node = self._insert_ctes( diff --git a/core/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py index fbf45d0c208..4b2bfa4b68d 100644 --- a/core/dbt/contracts/graph/compiled.py +++ b/core/dbt/contracts/graph/compiled.py @@ -43,6 +43,7 @@ class CompiledNode(ParsedNode, CompiledNodeMixin): extra_ctes_injected: bool = False extra_ctes: List[InjectedCTE] = field(default_factory=list) injected_sql: Optional[str] = None + relation_name: Optional[str] = None def set_cte(self, cte_id: str, sql: str): """This is the equivalent of what self.extra_ctes[cte_id] = sql would diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index 87d474a1cda..5899e9d364c 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -38,7 +38,7 @@ REQUIRED_COMPILED_NODE_KEYS = frozenset(REQUIRED_PARSED_NODE_KEYS | { 'compiled', 'extra_ctes_injected', 'extra_ctes', 'compiled_sql', - 'injected_sql', + 'injected_sql', 'relation_name' }) @@ -483,6 +483,7 @@ def setUp(self): compiled_sql='also does not matter', extra_ctes_injected=True, injected_sql=None, + relation_name='"dbt"."analytics"."events"', extra_ctes=[], checksum=FileHash.empty(), ), @@ -509,6 +510,7 @@ def setUp(self): compiled_sql='also does not matter', extra_ctes_injected=True, injected_sql='and this also does not matter', + relation_name='"dbt"."analytics"."events"', extra_ctes=[], checksum=FileHash.empty(), ), From 01331ed311e67052cffd79e7135a9fbb14ab45c4 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Fri, 16 Oct 2020 19:08:30 +0200 Subject: [PATCH 02/22] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e5030b85e0..d62b0ac79db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Added dbt_invocation_id for each BigQuery job to enable performance analysis ([#2808](https://github.com/fishtown-analytics/dbt/issues/2808), [#2809](https://github.com/fishtown-analytics/dbt/pull/2809)) - Save cli and rpc arguments in run_results.json ([#2510](https://github.com/fishtown-analytics/dbt/issues/2510), [#2813](https://github.com/fishtown-analytics/dbt/pull/2813)) - Added support for BigQuery connections using refresh tokens ([#2344](https://github.com/fishtown-analytics/dbt/issues/2344), [#2805](https://github.com/fishtown-analytics/dbt/pull/2805)) +- Store resolved node names in manifest ([#2647](https://github.com/fishtown-analytics/dbt/issues/2647), [#2837](https://github.com/fishtown-analytics/dbt/pull/2837)) ### Under the hood - Added strategy-specific validation to improve the relevancy of compilation errors for the `timestamp` and `check` snapshot strategies. (([#2787](https://github.com/fishtown-analytics/dbt/issues/2787), [#2791](https://github.com/fishtown-analytics/dbt/pull/2791)) @@ -24,6 +25,7 @@ Contributors: - [@joelluijmes](https://github.com/joelluijmes) ([#2749](https://github.com/fishtown-analytics/dbt/pull/2749), [#2821](https://github.com/fishtown-analytics/dbt/pull/2821)) - [@kingfink](https://github.com/kingfink) ([#2791](https://github.com/fishtown-analytics/dbt/pull/2791)) - [@zmac12](https://github.com/zmac12) ([#2871](https://github.com/fishtown-analytics/dbt/pull/2817)) +- [@franloza](https://github.com/franloza) ([#2837](https://github.com/fishtown-analytics/dbt/pull/2837)) ## dbt 0.18.1 (Release TBD) From a0370a66179b823d149cc69761e029ba7610854d Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Tue, 20 Oct 2020 18:22:32 +0200 Subject: [PATCH 03/22] Add relation_name to node object in docs generation tests --- .../test_docs_generate.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 994b64bed72..df018c6c941 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1048,6 +1048,9 @@ def expected_seeded_manifest(self, model_database=None): 'build_path': Normalized('target/compiled/test/models/model.sql'), 'name': 'model', 'root_path': self.test_root_realpath, + 'relation_name': '"{0}"."{1}".model'.format( + model_database, my_schema_name + ), 'resource_type': 'model', 'path': 'model.sql', 'original_file_path': model_sql_path, @@ -1122,6 +1125,9 @@ def expected_seeded_manifest(self, model_database=None): 'build_path': Normalized('target/compiled/test/models/second_model.sql'), 'name': 'second_model', 'root_path': self.test_root_realpath, + 'relation_name': '"{0}"."{1}".second_model'.format( + model_database, my_schema_name + ), 'resource_type': 'model', 'path': 'second_model.sql', 'original_file_path': second_model_sql_path, @@ -1201,6 +1207,9 @@ def expected_seeded_manifest(self, model_database=None): 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_realpath, + 'relation_name': '"{0}"."{1}".seed'.format( + model_database, my_schema_name + ), 'resource_type': 'seed', 'raw_sql': '', 'package_name': 'test', @@ -1289,6 +1298,9 @@ def expected_seeded_manifest(self, model_database=None): 'path': Normalized('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], + 'relation_name': '"{0}"."{1}".not_null_model_id'.format( + model_database, my_schema_name + ), 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -1334,6 +1346,9 @@ def expected_seeded_manifest(self, model_database=None): 'path': normalize('schema_test/test_nothing_model_.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], + 'relation_name': '"{0}"."{1}".test_nothing_model_'.format( + model_database, my_schema_name + ), 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -1378,6 +1393,9 @@ def expected_seeded_manifest(self, model_database=None): 'path': normalize('schema_test/unique_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], + 'relation_name': '"{0}"."{1}".unique_model_id'.format( + model_database, my_schema_name + ), 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -1576,6 +1594,9 @@ def expected_postgres_references_manifest(self, model_database=None): '\n\nselect * from {{ source("my_source", "my_table") }}' ), 'refs': [], + 'relation_name': '"{0}"."{1}".ephemeral_copy'.format( + model_database, my_schema_name + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -1634,6 +1655,9 @@ def expected_postgres_references_manifest(self, model_database=None): 'order by first_name asc' ), 'refs': [['ephemeral_copy']], + 'relation_name': '"{0}"."{1}".ephemeral_summary'.format( + model_database, my_schema_name + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -1765,6 +1789,9 @@ def expected_postgres_references_manifest(self, model_database=None): 'path': 'seed.csv', 'raw_sql': '', 'refs': [], + 'relation_name': '"{0}"."{1}".seed'.format( + model_database, my_schema_name + ), 'resource_type': 'seed', 'root_path': self.test_root_realpath, 'schema': my_schema_name, From 40370e104fa6d5bf9c30c098157786f6fd5daa62 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Tue, 20 Oct 2020 18:48:59 +0200 Subject: [PATCH 04/22] Fix wrong schema name in test and add missing relation_name in node --- .../029_docs_generate_tests/test_docs_generate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 34a01cffdd9..e96529e1c01 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1132,7 +1132,7 @@ def expected_seeded_manifest(self, model_database=None): 'name': 'second_model', 'root_path': self.test_root_realpath, 'relation_name': '"{0}"."{1}".second_model'.format( - model_database, my_schema_name + model_database, self.alternate_schema ), 'resource_type': 'model', 'path': 'second_model.sql', @@ -3242,6 +3242,9 @@ def expected_postgres_references_run_results(self): "{{ref('ephemeral_summary')}}\norder by ct asc" ), 'refs': [['ephemeral_summary']], + 'relation_name': '"{0}"."{1}".view_summary'.format( + self.default_database, my_schema_name + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, From 7ee78e89c9e09c86dd7993414c8ccf6be8cf10c9 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Tue, 20 Oct 2020 19:18:38 +0200 Subject: [PATCH 05/22] Add missing relation_name fields in doc generation test manifests --- .../test_docs_generate.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index e96529e1c01..eff9e996974 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1714,6 +1714,9 @@ def expected_postgres_references_manifest(self, model_database=None): "{{ref('ephemeral_summary')}}\norder by ct asc" ), 'refs': [['ephemeral_summary']], + 'relation_name': '"{0}"."{1}".view_summary'.format( + model_database, my_schema_name + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -2175,6 +2178,9 @@ def expected_bigquery_complex_manifest(self): 'path': 'nested_view.sql', 'raw_sql': LineIndifferent(_read_file(nested_view_sql_path).rstrip('\r\n')), 'refs': [['nested_table']], + 'relation_name': '"{0}"."{1}".nested_view'.format( + self.default_database, my_schema_name + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -2252,6 +2258,9 @@ def expected_bigquery_complex_manifest(self): 'path': 'nested_table.sql', 'raw_sql': LineIndifferent(_read_file(nested_table_sql_path).rstrip('\r\n')), 'refs': [], + 'relation_name': '"{0}"."{1}".nested_table'.format( + self.default_database, my_schema_name + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -2276,6 +2285,9 @@ def expected_bigquery_complex_manifest(self): 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_realpath, + 'relation_name': '"{0}"."{1}".seed'.format( + self.default_database, my_schema_name + ), 'resource_type': 'seed', 'raw_sql': '', 'package_name': 'test', @@ -2409,6 +2421,9 @@ def expected_redshift_incremental_view_manifest(self): 'build_path': Normalized('target/compiled/test/rs_models/model.sql'), 'name': 'model', 'root_path': self.test_root_realpath, + 'relation_name': '"{0}"."{1}".view_summary'.format( + self.default_database, my_schema_name + ), 'resource_type': 'model', 'path': 'model.sql', 'original_file_path': model_sql_path, @@ -2487,6 +2502,9 @@ def expected_redshift_incremental_view_manifest(self): 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_realpath, + 'relation_name': '"{0}"."{1}".seed'.format( + self.default_database, my_schema_name + ), 'resource_type': 'seed', 'raw_sql': '', 'package_name': 'test', @@ -2735,6 +2753,9 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': 'model.sql', 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), 'refs': [['seed']], + 'relation_name': '"{0}"."{1}".view_summary'.format( + model_database, schema + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': schema, @@ -2823,6 +2844,9 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': 'second_model.sql', 'raw_sql': LineIndifferent(_read_file(second_model_sql_path).rstrip('\r\n')), 'refs': [['seed']], + 'relation_name': '"{0}"."{1}".second_model'.format( + self.default_database, self.alternate_schema + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': self.alternate_schema, @@ -2906,6 +2930,9 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': 'seed.csv', 'raw_sql': '', 'refs': [], + 'relation_name': '"{0}"."{1}".view_summary'.format( + model_database, schema + ), 'resource_type': 'seed', 'root_path': self.test_root_realpath, 'schema': schema, @@ -3170,6 +3197,9 @@ def expected_postgres_references_run_results(self): 'order by first_name asc' ), 'refs': [['ephemeral_copy']], + 'relation_name': '"{0}"."{1}".ephemeral_summary'.format( + self.default_database, my_schema_name + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -3328,6 +3358,9 @@ def expected_postgres_references_run_results(self): 'path': 'seed.csv', 'raw_sql': '', 'refs': [], + 'relation_name': '"{0}"."{1}".seed'.format( + self.default_database, my_schema_name + ), 'resource_type': 'seed', 'root_path': self.test_root_realpath, 'schema': my_schema_name, From c3bf0f8cbf941db50b7c168964268d2121630d9d Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Sun, 25 Oct 2020 14:11:33 +0100 Subject: [PATCH 06/22] Add relation_name to missing tests in test_docs_generate --- .../029_docs_generate_tests/test_docs_generate.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index eff9e996974..d21cd13f5b3 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -2421,7 +2421,7 @@ def expected_redshift_incremental_view_manifest(self): 'build_path': Normalized('target/compiled/test/rs_models/model.sql'), 'name': 'model', 'root_path': self.test_root_realpath, - 'relation_name': '"{0}"."{1}".view_summary'.format( + 'relation_name': '"{0}"."{1}".model'.format( self.default_database, my_schema_name ), 'resource_type': 'model', @@ -2753,7 +2753,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': 'model.sql', 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), 'refs': [['seed']], - 'relation_name': '"{0}"."{1}".view_summary'.format( + 'relation_name': '"{0}"."{1}".model'.format( model_database, schema ), 'resource_type': 'model', @@ -2930,7 +2930,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': 'seed.csv', 'raw_sql': '', 'refs': [], - 'relation_name': '"{0}"."{1}".view_summary'.format( + 'relation_name': '"{0}"."{1}".seed'.format( model_database, schema ), 'resource_type': 'seed', @@ -2979,6 +2979,9 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': Normalized('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], + 'relation_name': '"{0}"."{1}".not_null_model_id'.format( + model_database, schema + ), 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': schema, @@ -3033,6 +3036,9 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': Normalized('schema_test/test_nothing_model_.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], + 'relation_name': '"{0}"."{1}".test_nothing_model_'.format( + model_database, schema + ), 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': schema, @@ -3086,6 +3092,9 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': Normalized('schema_test/unique_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], + 'relation_name': '"{0}"."{1}".unique_model_id'.format( + model_database, schema + ), 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': schema, From b079545e0f37669c6a059dd2a10bfc2a75c3bebb Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Sun, 25 Oct 2020 15:58:04 +0100 Subject: [PATCH 07/22] Adapt relation_name for Bigquery and Snowflake in docs generation tests --- .../test_docs_generate.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index d21cd13f5b3..74d9f78ecd1 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1047,6 +1047,10 @@ def expected_seeded_manifest(self, model_database=None): test_config = self.rendered_tst_config() unrendered_test_config = self.unrendered_tst_config() + relation_name_format = ( + '{0}.{1}."{2}"' if self.adapter_type == 'snowflake' + else '.'.join((self._quote("{0}"), self._quote("{1}"), "{2}"))) + return { 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', 'dbt_version': dbt.version.__version__, @@ -1055,8 +1059,8 @@ def expected_seeded_manifest(self, model_database=None): 'build_path': Normalized('target/compiled/test/models/model.sql'), 'name': 'model', 'root_path': self.test_root_realpath, - 'relation_name': '"{0}"."{1}".model'.format( - model_database, my_schema_name + 'relation_name': relation_name_format.format( + model_database, my_schema_name, 'model' ), 'resource_type': 'model', 'path': 'model.sql', @@ -1131,8 +1135,8 @@ def expected_seeded_manifest(self, model_database=None): 'build_path': Normalized('target/compiled/test/models/second_model.sql'), 'name': 'second_model', 'root_path': self.test_root_realpath, - 'relation_name': '"{0}"."{1}".second_model'.format( - model_database, self.alternate_schema + 'relation_name': relation_name_format.format( + model_database, self.alternate_schema, 'second_model' ), 'resource_type': 'model', 'path': 'second_model.sql', @@ -1212,8 +1216,8 @@ def expected_seeded_manifest(self, model_database=None): 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_realpath, - 'relation_name': '"{0}"."{1}".seed'.format( - model_database, my_schema_name + 'relation_name': relation_name_format.format( + model_database, my_schema_name, 'seed' ), 'resource_type': 'seed', 'raw_sql': '', @@ -1302,8 +1306,8 @@ def expected_seeded_manifest(self, model_database=None): 'path': Normalized('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': '"{0}"."{1}".not_null_model_id'.format( - model_database, my_schema_name + 'relation_name': relation_name_format.format( + model_database, my_schema_name, 'not_null_model_id' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, @@ -1349,8 +1353,8 @@ def expected_seeded_manifest(self, model_database=None): 'path': normalize('schema_test/test_nothing_model_.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': '"{0}"."{1}".test_nothing_model_'.format( - model_database, my_schema_name + 'relation_name': relation_name_format.format( + model_database, my_schema_name, 'test_nothing_model_' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, @@ -1395,8 +1399,8 @@ def expected_seeded_manifest(self, model_database=None): 'path': normalize('schema_test/unique_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': '"{0}"."{1}".unique_model_id'.format( - model_database, my_schema_name + 'relation_name': relation_name_format.format( + model_database, my_schema_name, 'unique_model_id' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, From c9e01bcc819cc4b00cfa867babc8856b54630ee4 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Sun, 25 Oct 2020 16:31:00 +0100 Subject: [PATCH 08/22] Fix quotes in relation name for Bigquery docs generate tests --- .../test_docs_generate.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 74d9f78ecd1..70f38fe060b 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1136,7 +1136,8 @@ def expected_seeded_manifest(self, model_database=None): 'name': 'second_model', 'root_path': self.test_root_realpath, 'relation_name': relation_name_format.format( - model_database, self.alternate_schema, 'second_model' + self.default_database, self.alternate_schema, + 'second_model' ), 'resource_type': 'model', 'path': 'second_model.sql', @@ -1217,7 +1218,7 @@ def expected_seeded_manifest(self, model_database=None): 'name': 'seed', 'root_path': self.test_root_realpath, 'relation_name': relation_name_format.format( - model_database, my_schema_name, 'seed' + self.default_database, my_schema_name, 'seed' ), 'resource_type': 'seed', 'raw_sql': '', @@ -1307,7 +1308,8 @@ def expected_seeded_manifest(self, model_database=None): 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'relation_name': relation_name_format.format( - model_database, my_schema_name, 'not_null_model_id' + self.default_database, my_schema_name, + 'not_null_model_id' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, @@ -1354,7 +1356,8 @@ def expected_seeded_manifest(self, model_database=None): 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'relation_name': relation_name_format.format( - model_database, my_schema_name, 'test_nothing_model_' + self.default_database, my_schema_name, + 'test_nothing_model_' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, @@ -1400,7 +1403,7 @@ def expected_seeded_manifest(self, model_database=None): 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'relation_name': relation_name_format.format( - model_database, my_schema_name, 'unique_model_id' + self.default_database, my_schema_name, 'unique_model_id' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, @@ -2021,6 +2024,9 @@ def expected_bigquery_complex_manifest(self): 'path': 'clustered.sql', 'raw_sql': LineIndifferent(_read_file(clustered_sql_path).rstrip('\r\n')), 'refs': [['seed']], + 'relation_name': '`{0}`.`{1}`.clustered'.format( + self.default_database, my_schema_name + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -2102,6 +2108,9 @@ def expected_bigquery_complex_manifest(self): 'path': 'multi_clustered.sql', 'raw_sql': LineIndifferent(_read_file(multi_clustered_sql_path).rstrip('\r\n')), 'refs': [['seed']], + 'relation_name': '`{0}`.`{1}`.multi_clustered'.format( + self.default_database, my_schema_name + ), 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -2182,7 +2191,7 @@ def expected_bigquery_complex_manifest(self): 'path': 'nested_view.sql', 'raw_sql': LineIndifferent(_read_file(nested_view_sql_path).rstrip('\r\n')), 'refs': [['nested_table']], - 'relation_name': '"{0}"."{1}".nested_view'.format( + 'relation_name': '`{0}`.`{1}`.nested_view'.format( self.default_database, my_schema_name ), 'resource_type': 'model', @@ -2262,7 +2271,7 @@ def expected_bigquery_complex_manifest(self): 'path': 'nested_table.sql', 'raw_sql': LineIndifferent(_read_file(nested_table_sql_path).rstrip('\r\n')), 'refs': [], - 'relation_name': '"{0}"."{1}".nested_table'.format( + 'relation_name': '`{0}`.`{1}`.nested_table'.format( self.default_database, my_schema_name ), 'resource_type': 'model', @@ -2289,7 +2298,7 @@ def expected_bigquery_complex_manifest(self): 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_realpath, - 'relation_name': '"{0}"."{1}".seed'.format( + 'relation_name': '`{0}`.`{1}`.seed'.format( self.default_database, my_schema_name ), 'resource_type': 'seed', From 09c37f508e820ecb17b9204dbcdb0c1cefe8c6ba Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Sun, 25 Oct 2020 17:27:46 +0100 Subject: [PATCH 09/22] Adapt relation_name to expected_run_results parameters --- .../test_docs_generate.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 70f38fe060b..67aa0c6fbd1 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1048,7 +1048,7 @@ def expected_seeded_manifest(self, model_database=None): unrendered_test_config = self.unrendered_tst_config() relation_name_format = ( - '{0}.{1}."{2}"' if self.adapter_type == 'snowflake' + '{0}.{1}.{2}' if self.adapter_type == 'snowflake' else '.'.join((self._quote("{0}"), self._quote("{1}"), "{2}"))) return { @@ -2674,11 +2674,16 @@ def expected_run_results(self, quote_schema=True, quote_model=False, schema = self.unique_schema() # we are selecting from the seed, which is always in the default db - compiled_database = self.default_database - if self.adapter_type != 'snowflake': - compiled_database = self._quote(compiled_database) + quote_database = self.adapter_type != 'snowflake' + compiled_database = (self._quote(self.default_database) + if quote_database else self.default_database) compiled_schema = self._quote(schema) if quote_schema else schema compiled_seed = self._quote('seed') if quote_model else 'seed' + relation_name_format = ".".join(( + self._quote("{0}") if quote_database else '{0}', + self._quote("{1}") if quote_schema else '{1}', + self._quote("{2}") if quote_model else '{2}', + )) if self.adapter_type == 'bigquery': compiled_sql = '\n\nselect * from `{}`.`{}`.seed'.format( @@ -2766,8 +2771,8 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': 'model.sql', 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), 'refs': [['seed']], - 'relation_name': '"{0}"."{1}".model'.format( - model_database, schema + 'relation_name': relation_name_format.format( + model_database, schema, 'model' ), 'resource_type': 'model', 'root_path': self.test_root_realpath, @@ -2857,8 +2862,9 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': 'second_model.sql', 'raw_sql': LineIndifferent(_read_file(second_model_sql_path).rstrip('\r\n')), 'refs': [['seed']], - 'relation_name': '"{0}"."{1}".second_model'.format( - self.default_database, self.alternate_schema + 'relation_name': relation_name_format.format( + self.default_database, self.alternate_schema, + 'second_model' ), 'resource_type': 'model', 'root_path': self.test_root_realpath, @@ -2943,8 +2949,8 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': 'seed.csv', 'raw_sql': '', 'refs': [], - 'relation_name': '"{0}"."{1}".seed'.format( - model_database, schema + 'relation_name': relation_name_format.format( + model_database, schema, 'seed' ), 'resource_type': 'seed', 'root_path': self.test_root_realpath, @@ -2992,8 +2998,8 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': Normalized('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': '"{0}"."{1}".not_null_model_id'.format( - model_database, schema + 'relation_name': relation_name_format.format( + model_database, schema, 'not_null_model_id' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, @@ -3049,8 +3055,8 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': Normalized('schema_test/test_nothing_model_.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': '"{0}"."{1}".test_nothing_model_'.format( - model_database, schema + 'relation_name': relation_name_format.format( + model_database, schema, 'test_nothing_model_' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, @@ -3105,8 +3111,8 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': Normalized('schema_test/unique_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': '"{0}"."{1}".unique_model_id'.format( - model_database, schema + 'relation_name': relation_name_format.format( + model_database, schema, 'unique_model_id' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, From 900298bce7ffde47c75363bf7c88d453c306ad22 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Sun, 25 Oct 2020 18:06:18 +0100 Subject: [PATCH 10/22] Fix database name in relation_name in expected_run_results --- .../029_docs_generate_tests/test_docs_generate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 67aa0c6fbd1..f07ab3981c3 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1048,7 +1048,7 @@ def expected_seeded_manifest(self, model_database=None): unrendered_test_config = self.unrendered_tst_config() relation_name_format = ( - '{0}.{1}.{2}' if self.adapter_type == 'snowflake' + '{0}.{1}."{2}"' if self.adapter_type == 'snowflake' else '.'.join((self._quote("{0}"), self._quote("{1}"), "{2}"))) return { @@ -2950,7 +2950,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'raw_sql': '', 'refs': [], 'relation_name': relation_name_format.format( - model_database, schema, 'seed' + self.default_database, schema, 'seed' ), 'resource_type': 'seed', 'root_path': self.test_root_realpath, @@ -2999,7 +2999,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'relation_name': relation_name_format.format( - model_database, schema, 'not_null_model_id' + self.default_database, schema, 'not_null_model_id' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, @@ -3056,7 +3056,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'relation_name': relation_name_format.format( - model_database, schema, 'test_nothing_model_' + self.default_database, schema, 'test_nothing_model_' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, @@ -3112,7 +3112,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'relation_name': relation_name_format.format( - model_database, schema, 'unique_model_id' + self.default_database, schema, 'unique_model_id' ), 'resource_type': 'test', 'root_path': self.test_root_realpath, From 4203985e3e1ebfcc8c3e9c3502dc60f7c1eed733 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Sun, 25 Oct 2020 18:50:52 +0100 Subject: [PATCH 11/22] Adapt expected_seeded_manifest method to Snowflake identifier quoting --- .../029_docs_generate_tests/test_docs_generate.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index f07ab3981c3..6b75ee6bed2 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1022,7 +1022,7 @@ def _verify_generic_macro_structure(self, manifest): without_sql, ) - def expected_seeded_manifest(self, model_database=None): + def expected_seeded_manifest(self, model_database=None, quote_model=False): models_path = self.dir('models') model_sql_path = os.path.join(models_path, 'model.sql') second_model_sql_path = os.path.join(models_path, 'second_model.sql') @@ -1047,9 +1047,12 @@ def expected_seeded_manifest(self, model_database=None): test_config = self.rendered_tst_config() unrendered_test_config = self.unrendered_tst_config() - relation_name_format = ( - '{0}.{1}."{2}"' if self.adapter_type == 'snowflake' - else '.'.join((self._quote("{0}"), self._quote("{1}"), "{2}"))) + quote_database = quote_schema = self.adapter_type != 'snowflake' + relation_name_format = ".".join(( + self._quote("{0}") if quote_database else '{0}', + self._quote("{1}") if quote_schema else '{1}', + self._quote("{2}") if quote_model else '{2}', + )) return { 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', @@ -3495,7 +3498,7 @@ def connect(*args, **kwargs): }) self.verify_catalog(self.expected_snowflake_catalog(case_columns=True)) - self.verify_manifest(self.expected_seeded_manifest()) + self.verify_manifest(self.expected_seeded_manifest(quote_model=True)) self.verify_run_results(self.expected_run_results(quote_schema=False, quote_model=True)) @use_profile('bigquery') From e1097f11b580da1ef2a4480be178338b17560b31 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Tue, 27 Oct 2020 17:21:45 +0100 Subject: [PATCH 12/22] Define relation_name only for non-ephemeral models, seeds and snapshots --- core/dbt/compilation.py | 10 +- .../snapshots/snapshot_seed.sql | 11 ++ .../test_docs_generate.py | 174 +++++++++++++++--- 3 files changed, 168 insertions(+), 27 deletions(-) create mode 100644 test/integration/029_docs_generate_tests/snapshots/snapshot_seed.sql diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 4f7a590a73d..f745ae908b2 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -173,9 +173,13 @@ def add_ephemeral_prefix(self, name: str): return relation_cls.add_ephemeral_prefix(name) def _get_relation_name(self, node: ParsedNode): - adapter = get_adapter(self.config) - relation_cls = adapter.Relation - return str(relation_cls.create_from(self.config, node)) + relation_name = None + if (node.resource_type in NodeType.refable() + and node.config.materialized != "ephemeral"): + adapter = get_adapter(self.config) + relation_cls = adapter.Relation + relation_name = str(relation_cls.create_from(self.config, node)) + return relation_name def _inject_ctes_into_sql(self, sql: str, ctes: List[InjectedCTE]) -> str: """ diff --git a/test/integration/029_docs_generate_tests/snapshots/snapshot_seed.sql b/test/integration/029_docs_generate_tests/snapshots/snapshot_seed.sql new file mode 100644 index 00000000000..83bc54fe784 --- /dev/null +++ b/test/integration/029_docs_generate_tests/snapshots/snapshot_seed.sql @@ -0,0 +1,11 @@ +{% snapshot snapshot_seed %} +{{ + config( + unique_key='id', + strategy='check', + check_cols='all', + target_schema=var('alternate_schema') + ) +}} +select * from {{ ref('seed') }} +{% endsnapshot %} \ No newline at end of file diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 6b75ee6bed2..7cfa087c0f1 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -156,6 +156,7 @@ def run_and_generate(self, extra=None, seed_count=1, model_count=2, alternate_db project = { "data-paths": [self.dir("seed")], 'macro-paths': [self.dir('macros')], + 'snapshot-paths': [self.dir('snapshots')], 'vars': { 'alternate_db': alternate_db, 'alternate_schema': self.alternate_schema, @@ -950,6 +951,39 @@ def unrendered_seed_config(self, **updates): result.update(updates) return result + def rendered_snapshot_config(self, **updates): + result = { + 'database': None, + 'schema': None, + 'alias': None, + 'enabled': True, + 'materialized': 'snapshot', + 'pre-hook': [], + 'post-hook': [], + 'vars': {}, + 'column_types': {}, + 'quoting': {}, + 'tags': [], + 'persist_docs': {}, + 'full_refresh': None, + 'strategy': 'check', + 'check_cols': 'all', + 'unique_key': 'id', + 'target_schema': None + } + result.update(updates) + return result + + def unrendered_snapshot_config(self, **updates): + result = { + 'check_cols': 'all', + 'strategy': 'check', + 'target_schema': None, + 'unique_key': 'id' + } + result.update(updates) + return result + def rendered_tst_config(self, **updates): result = { 'column_types': {}, @@ -1029,6 +1063,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): model_schema_yml_path = os.path.join(models_path, 'schema.yml') seed_schema_yml_path = os.path.join(self.dir('seed'), 'schema.yml') seed_path = self.dir(os.path.join('seed', 'seed.csv')) + snapshot_path = self.dir(os.path.join('snapshots', 'snapshot_seed.sql')) my_schema_name = self.unique_schema() @@ -1047,6 +1082,12 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): test_config = self.rendered_tst_config() unrendered_test_config = self.unrendered_tst_config() + snapshot_config = self.rendered_snapshot_config( + target_schema=self.alternate_schema) + unrendered_snapshot_config = self.unrendered_snapshot_config( + target_schema=self.alternate_schema + ) + quote_database = quote_schema = self.adapter_type != 'snowflake' relation_name_format = ".".join(( self._quote("{0}") if quote_database else '{0}', @@ -1310,10 +1351,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'path': Normalized('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': relation_name_format.format( - self.default_database, my_schema_name, - 'not_null_model_id' - ), + 'relation_name': None, 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -1337,6 +1375,47 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'checksum': {'name': 'none', 'checksum': ''}, 'unrendered_config': unrendered_test_config, }, + 'snapshot.test.snapshot_seed': { + 'alias': 'snapshot_seed', + 'build_path': None, + 'checksum': self._checksum_file(snapshot_path), + 'columns': {}, + 'compiled': True, + 'compiled_sql': ANY, + 'config': snapshot_config, + 'database': model_database, + 'deferred': False, + 'depends_on': { + 'macros': [], + 'nodes': ['seed.test.seed'], + }, + 'description': '', + 'docs': {'show': True}, + 'extra_ctes': [], + 'extra_ctes_injected': True, + 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], + 'meta': {}, + 'name': 'snapshot_seed', + 'original_file_path': snapshot_path, + 'package_name': 'test', + 'patch_path': None, + 'path': normalize('snapshot_seed.sql'), + 'raw_sql': LineIndifferent( + _read_file(snapshot_path) + .replace('{% snapshot snapshot_seed %}', '') + .replace('{% endsnapshot %}', '')), + 'refs': [['seed']], + 'relation_name': relation_name_format.format( + model_database, self.alternate_schema, 'snapshot_seed' + ), + 'resource_type': 'snapshot', + 'root_path': self.test_root_realpath, + 'schema': self.alternate_schema, + 'sources': [], + 'tags': [], + 'unique_id': 'snapshot.test.snapshot_seed', + 'unrendered_config': unrendered_snapshot_config, + }, 'test.test.test_nothing_model_': { 'alias': 'test_nothing_model_', 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/test_nothing_model_.sql'), @@ -1358,10 +1437,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'path': normalize('schema_test/test_nothing_model_.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': relation_name_format.format( - self.default_database, my_schema_name, - 'test_nothing_model_' - ), + 'relation_name': None, 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -1405,9 +1481,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'path': normalize('schema_test/unique_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': relation_name_format.format( - self.default_database, my_schema_name, 'unique_model_id' - ), + 'relation_name': None, 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -1537,6 +1611,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'exposure.test.notebook_exposure': ['model.test.model', 'model.test.second_model'], 'exposure.test.simple_exposure': ['model.test.model', 'source.test.my_source.my_table'], 'seed.test.seed': [], + 'snapshot.test.snapshot_seed': ['seed.test.seed'], 'source.test.my_source.my_table': [], 'test.test.not_null_model_id': ['model.test.model'], 'test.test.test_nothing_model_': ['model.test.model'], @@ -1553,7 +1628,10 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'model.test.second_model': ['exposure.test.notebook_exposure'], 'exposure.test.notebook_exposure': [], 'exposure.test.simple_exposure': [], - 'seed.test.seed': ['model.test.model', 'model.test.second_model'], + 'seed.test.seed': ['model.test.model', + 'model.test.second_model', + 'snapshot.test.snapshot_seed'], + 'snapshot.test.snapshot_seed': [], 'source.test.my_source.my_table': ['exposure.test.simple_exposure'], 'test.test.not_null_model_id': [], 'test.test.test_nothing_model_': [], @@ -1605,9 +1683,7 @@ def expected_postgres_references_manifest(self, model_database=None): '\n\nselect * from {{ source("my_source", "my_table") }}' ), 'refs': [], - 'relation_name': '"{0}"."{1}".ephemeral_copy'.format( - model_database, my_schema_name - ), + 'relation_name': None, 'resource_type': 'model', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -2697,6 +2773,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, compiled_database, compiled_schema, compiled_seed ) seed_path = self.dir('seed/seed.csv') + snapshot_path = self.dir('snapshots/snapshot_seed.sql') return [ { @@ -2968,6 +3045,61 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'skip': False, 'status': None, }, + { + 'error': None, + 'warn': None, + 'execution_time': AnyFloat(), + 'fail': None, + 'node': { + 'alias': 'snapshot_seed', + 'build_path': None, + 'checksum': self._checksum_file(snapshot_path), + 'columns': {}, + 'compiled': True, + 'compiled_sql': compiled_sql, + 'config': self.rendered_snapshot_config( + target_schema=self.alternate_schema + ), + 'database': self.default_database, + 'deferred': False, + 'depends_on': { + 'macros': [], + 'nodes': ['seed.test.seed'], + }, + 'description': '', + 'docs': {'show': True}, + 'extra_ctes': [], + 'extra_ctes_injected': True, + 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], + 'meta': {}, + 'name': 'snapshot_seed', + 'original_file_path': snapshot_path, + 'package_name': 'test', + 'patch_path': None, + 'path': Normalized('snapshot_seed.sql'), + 'raw_sql': LineIndifferent( + _read_file(snapshot_path) + .replace('{% snapshot snapshot_seed %}', '') + .replace('{% endsnapshot %}', '')), + 'refs': [['seed']], + 'relation_name': relation_name_format.format( + self.default_database, self.alternate_schema, + 'snapshot_seed'), + 'resource_type': 'snapshot', + 'root_path': self.test_root_realpath, + 'schema': self.alternate_schema, + 'sources': [], + 'tags': [], + 'unique_id': 'snapshot.test.snapshot_seed', + 'unrendered_config': self.unrendered_snapshot_config( + target_schema=self.alternate_schema + ), + }, + 'thread_id': ANY, + 'timing': [ANY, ANY], + 'skip': False, + 'status': None, + }, { 'error': None, 'execution_time': AnyFloat(), @@ -3001,9 +3133,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': Normalized('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': relation_name_format.format( - self.default_database, schema, 'not_null_model_id' - ), + 'relation_name': None, 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': schema, @@ -3058,9 +3188,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': Normalized('schema_test/test_nothing_model_.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': relation_name_format.format( - self.default_database, schema, 'test_nothing_model_' - ), + 'relation_name': None, 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': schema, @@ -3114,9 +3242,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'path': Normalized('schema_test/unique_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], - 'relation_name': relation_name_format.format( - self.default_database, schema, 'unique_model_id' - ), + 'relation_name': None, 'resource_type': 'test', 'root_path': self.test_root_realpath, 'schema': schema, From 92cedf8931f858c30b9932e7f42a4b529e70ba18 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Tue, 27 Oct 2020 17:39:20 +0100 Subject: [PATCH 13/22] Fix Flake8 style issue --- core/dbt/compilation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index f745ae908b2..02e4d31ae6d 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -174,8 +174,8 @@ def add_ephemeral_prefix(self, name: str): def _get_relation_name(self, node: ParsedNode): relation_name = None - if (node.resource_type in NodeType.refable() - and node.config.materialized != "ephemeral"): + if (node.resource_type in NodeType.refable() and + node.config.materialized != "ephemeral"): adapter = get_adapter(self.config) relation_cls = adapter.Relation relation_name = str(relation_cls.create_from(self.config, node)) From 52ed4aa6314a90ef3f4f53e435cdba172b920039 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Tue, 27 Oct 2020 18:45:00 +0100 Subject: [PATCH 14/22] Fix tests which are missing snapshot nodes --- .../test_docs_generate.py | 109 +++++++++++++++++- 1 file changed, 103 insertions(+), 6 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 7cfa087c0f1..7c31215851f 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1654,6 +1654,7 @@ def expected_postgres_references_manifest(self, model_database=None): ephemeral_summary_path = self.dir('ref_models/ephemeral_summary.sql') view_summary_path = self.dir('ref_models/view_summary.sql') seed_path = self.dir('seed/seed.csv') + snapshot_path = self.dir('snapshots/snapshot_seed.sql') return { 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', @@ -1893,6 +1894,44 @@ def expected_postgres_references_manifest(self, model_database=None): 'checksum': self._checksum_file(seed_path), 'unrendered_config': self.unrendered_seed_config(), }, + 'snapshot.test.snapshot_seed': { + 'alias': 'snapshot_seed', + 'build_path': None, + 'checksum': self._checksum_file(snapshot_path), + 'columns': {}, + 'compiled': True, + 'compiled_sql': ANY, + 'config': self.rendered_snapshot_config( + target_schema=self.alternate_schema + ), + 'database': model_database, + 'deferred': False, + 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, + 'description': '', + 'docs': {'show': True}, + 'extra_ctes': [], + 'extra_ctes_injected': True, + 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], + 'meta': {}, + 'name': 'snapshot_seed', + 'original_file_path': snapshot_path, + 'package_name': 'test', + 'patch_path': None, + 'path': 'snapshot_seed.sql', + 'raw_sql': ANY, + 'refs': [['seed']], + 'relation_name': '"{0}"."{1}".snapshot_seed'.format( + model_database, self.alternate_schema + ), + 'resource_type': 'snapshot', + 'root_path': self.test_root_realpath, + 'schema': self.alternate_schema, + 'sources': [], + 'tags': [], + 'unique_id': 'snapshot.test.snapshot_seed', + 'unrendered_config': self.unrendered_snapshot_config( + target_schema=self.alternate_schema + )} }, 'sources': { 'source.test.my_source.my_table': { @@ -2034,7 +2073,8 @@ def expected_postgres_references_manifest(self, model_database=None): 'model.test.ephemeral_copy': ['model.test.ephemeral_summary'], 'model.test.ephemeral_summary': ['model.test.view_summary'], 'model.test.view_summary': [], - 'seed.test.seed': [], + 'seed.test.seed': ['snapshot.test.snapshot_seed'], + 'snapshot.test.snapshot_seed': [], 'source.test.my_source.my_table': ['model.test.ephemeral_copy'], }, 'parent_map': { @@ -2042,6 +2082,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'model.test.ephemeral_summary': ['model.test.ephemeral_copy'], 'model.test.view_summary': ['model.test.ephemeral_summary'], 'seed.test.seed': [], + 'snapshot.test.snapshot_seed': ['seed.test.seed'], 'source.test.my_source.my_table': [], }, 'disabled': [], @@ -3268,11 +3309,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, def expected_postgres_references_run_results(self): my_schema_name = self.unique_schema() - ephemeral_compiled_sql = ( - '\n\nselect first_name, count(*) as ct from ' - '__dbt__CTE__ephemeral_copy\ngroup by first_name\n' - 'order by first_name asc' - ) cte_sql = ( ' __dbt__CTE__ephemeral_copy as (\n\n\nselect * from {}."{}"."seed"\n)' @@ -3289,9 +3325,14 @@ def expected_postgres_references_run_results(self): 'order by ct asc' ).format(self.default_database, my_schema_name) + snapshot_compiled_sql = ( + '\n\nselect * from "{}"."{}".seed' + ).format(self.default_database, my_schema_name) + ephemeral_summary_path = self.dir('ref_models/ephemeral_summary.sql') view_summary_path = self.dir('ref_models/view_summary.sql') seed_path = self.dir('seed/seed.csv') + snapshot_path = self.dir('snapshots/snapshot_seed.sql') return [ { @@ -3531,6 +3572,58 @@ def expected_postgres_references_run_results(self): 'skip': False, 'status': None, }, + { + 'error': None, + 'warn': None, + 'execution_time': AnyFloat(), + 'fail': None, + 'node': { + 'alias': 'snapshot_seed', + 'build_path': None, + 'checksum': self._checksum_file(snapshot_path), + 'columns': {}, + 'compiled': True, + 'compiled_sql': snapshot_compiled_sql, + 'config': self.rendered_snapshot_config( + target_schema=self.alternate_schema + ), + 'database': self.default_database, + 'deferred': False, + 'depends_on': { + 'macros': [], + 'nodes': ['seed.test.seed'], + }, + 'description': '', + 'docs': {'show': True}, + 'extra_ctes': [], + 'extra_ctes_injected': True, + 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], + 'meta': {}, + 'name': 'snapshot_seed', + 'original_file_path': snapshot_path, + 'package_name': 'test', + 'patch_path': None, + 'path': Normalized('snapshot_seed.sql'), + 'raw_sql': ANY, + 'refs': [['seed']], + 'relation_name': '"{0}"."{1}".snapshot_seed'.format( + self.default_database, self.alternate_schema + ), + 'resource_type': 'snapshot', + 'root_path': self.test_root_realpath, + 'schema': self.alternate_schema, + 'sources': [], + 'tags': [], + 'unique_id': 'snapshot.test.snapshot_seed', + 'unrendered_config': self.unrendered_snapshot_config( + target_schema=self.alternate_schema + ), + }, + 'thread_id': ANY, + 'timing': [ANY, ANY], + 'skip': False, + 'status': None, + }, ] def verify_run_results(self, expected_run_results): @@ -3727,6 +3820,10 @@ def project_config(self): return { 'config-version': 2, 'macro-paths': [self.dir('fail_macros')], + 'data-paths': [self.dir("seed")], + 'vars': { + 'alternate_schema': self.schema, + }, } @use_profile('postgres') From 7115d862ea6b369bf5a6932e3b531fc77c416068 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Tue, 27 Oct 2020 18:59:32 +0100 Subject: [PATCH 15/22] Modify snapshot path for docs generation tests --- .../{snapshots => snapshot}/snapshot_seed.sql | 0 .../029_docs_generate_tests/test_docs_generate.py | 14 +++++--------- 2 files changed, 5 insertions(+), 9 deletions(-) rename test/integration/029_docs_generate_tests/{snapshots => snapshot}/snapshot_seed.sql (100%) diff --git a/test/integration/029_docs_generate_tests/snapshots/snapshot_seed.sql b/test/integration/029_docs_generate_tests/snapshot/snapshot_seed.sql similarity index 100% rename from test/integration/029_docs_generate_tests/snapshots/snapshot_seed.sql rename to test/integration/029_docs_generate_tests/snapshot/snapshot_seed.sql diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 7c31215851f..36c1c05b6a2 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -156,7 +156,7 @@ def run_and_generate(self, extra=None, seed_count=1, model_count=2, alternate_db project = { "data-paths": [self.dir("seed")], 'macro-paths': [self.dir('macros')], - 'snapshot-paths': [self.dir('snapshots')], + 'snapshot-paths': [self.dir('snapshot')], 'vars': { 'alternate_db': alternate_db, 'alternate_schema': self.alternate_schema, @@ -1063,7 +1063,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): model_schema_yml_path = os.path.join(models_path, 'schema.yml') seed_schema_yml_path = os.path.join(self.dir('seed'), 'schema.yml') seed_path = self.dir(os.path.join('seed', 'seed.csv')) - snapshot_path = self.dir(os.path.join('snapshots', 'snapshot_seed.sql')) + snapshot_path = self.dir(os.path.join('snapshot', 'snapshot_seed.sql')) my_schema_name = self.unique_schema() @@ -1654,7 +1654,7 @@ def expected_postgres_references_manifest(self, model_database=None): ephemeral_summary_path = self.dir('ref_models/ephemeral_summary.sql') view_summary_path = self.dir('ref_models/view_summary.sql') seed_path = self.dir('seed/seed.csv') - snapshot_path = self.dir('snapshots/snapshot_seed.sql') + snapshot_path = self.dir('snapshot/snapshot_seed.sql') return { 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', @@ -2814,7 +2814,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, compiled_database, compiled_schema, compiled_seed ) seed_path = self.dir('seed/seed.csv') - snapshot_path = self.dir('snapshots/snapshot_seed.sql') + snapshot_path = self.dir('snapshot/snapshot_seed.sql') return [ { @@ -3332,7 +3332,7 @@ def expected_postgres_references_run_results(self): ephemeral_summary_path = self.dir('ref_models/ephemeral_summary.sql') view_summary_path = self.dir('ref_models/view_summary.sql') seed_path = self.dir('seed/seed.csv') - snapshot_path = self.dir('snapshots/snapshot_seed.sql') + snapshot_path = self.dir('snapshot/snapshot_seed.sql') return [ { @@ -3820,10 +3820,6 @@ def project_config(self): return { 'config-version': 2, 'macro-paths': [self.dir('fail_macros')], - 'data-paths': [self.dir("seed")], - 'vars': { - 'alternate_schema': self.schema, - }, } @use_profile('postgres') From a9901c4ea7f525a9f43341b861a93835382ebb49 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Tue, 27 Oct 2020 19:27:54 +0100 Subject: [PATCH 16/22] Disable snapshot documentation testing for Redshift and Bigquery --- .../029_docs_generate_tests/test_docs_generate.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 36c1c05b6a2..f4dd6917579 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1383,7 +1383,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'compiled': True, 'compiled_sql': ANY, 'config': snapshot_config, - 'database': model_database, + 'database': self.default_database, 'deferred': False, 'depends_on': { 'macros': [], @@ -1406,7 +1406,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): .replace('{% endsnapshot %}', '')), 'refs': [['seed']], 'relation_name': relation_name_format.format( - model_database, self.alternate_schema, 'snapshot_seed' + self.default_database, self.alternate_schema, 'snapshot_seed' ), 'resource_type': 'snapshot', 'root_path': self.test_root_realpath, @@ -3731,7 +3731,8 @@ def test__bigquery__run_and_generate(self): @use_profile('bigquery') def test__bigquery__complex_models(self): self.run_and_generate( - extra={'source-paths': [self.dir('bq_models')]}, + extra={'source-paths': [self.dir('bq_models')], + 'snapshot-paths': []}, model_count=4 ) @@ -3752,7 +3753,7 @@ def test__redshift__run_and_generate(self): @use_profile('redshift') def test__redshift__incremental_view(self): self.run_and_generate( - {'source-paths': [self.dir('rs_models')]}, + {'source-paths': [self.dir('rs_models')], 'snapshot-paths': []}, alternate_db=self.default_database, model_count=1, ) From 6251d199462152e9478308bd964af978cead5b12 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Wed, 28 Oct 2020 09:49:44 +0100 Subject: [PATCH 17/22] Use is_ephemeral_model property instead of config.materialized Co-authored-by: Jeremy Cohen --- core/dbt/compilation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 02e4d31ae6d..72b0251ca70 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -175,7 +175,7 @@ def add_ephemeral_prefix(self, name: str): def _get_relation_name(self, node: ParsedNode): relation_name = None if (node.resource_type in NodeType.refable() and - node.config.materialized != "ephemeral"): + not node.is_ephemeral_model adapter = get_adapter(self.config) relation_cls = adapter.Relation relation_name = str(relation_cls.create_from(self.config, node)) From 784616ec295c327ea2a838a355e2bd7704687878 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Wed, 28 Oct 2020 18:58:25 +0100 Subject: [PATCH 18/22] Add relation name to source object in manifest --- core/dbt/compilation.py | 2 +- core/dbt/contracts/graph/parsed.py | 1 + core/dbt/parser/schemas.py | 12 ++++- .../test_docs_generate.py | 44 ++++++++++++------- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 72b0251ca70..f66b6806a9c 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -175,7 +175,7 @@ def add_ephemeral_prefix(self, name: str): def _get_relation_name(self, node: ParsedNode): relation_name = None if (node.resource_type in NodeType.refable() and - not node.is_ephemeral_model + not node.is_ephemeral_model): adapter = get_adapter(self.config) relation_cls = adapter.Relation relation_name = str(relation_cls.create_from(self.config, node)) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index a7c457c7a19..422c78f5390 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -555,6 +555,7 @@ class ParsedSourceDefinition( config: SourceConfig = field(default_factory=SourceConfig) patch_path: Optional[Path] = None unrendered_config: Dict[str, Any] = field(default_factory=dict) + relation_name: Optional[str] = None def same_database_representation( self, other: 'ParsedSourceDefinition' diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 2fbb7815ac9..74c7228e424 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -264,6 +264,11 @@ def _generate_source_config(self, fqn: List[str], rendered: bool): base=False, ) + def _get_relation_name(self, node: ParsedSourceDefinition): + adapter = get_adapter(self.root_project) + relation_cls = adapter.Relation + return str(relation_cls.create_from(self.root_project, node)) + def parse_source( self, target: UnpatchedSourceDefinition ) -> ParsedSourceDefinition: @@ -302,7 +307,7 @@ def parse_source( default_database = self.root_project.credentials.database - return ParsedSourceDefinition( + parsed_source = ParsedSourceDefinition( package_name=target.package_name, database=(source.database or default_database), schema=(source.schema or source.name), @@ -330,6 +335,11 @@ def parse_source( unrendered_config=unrendered_config, ) + # relation name is added after instantiation because the adapter does + # not provide the relation name for a UnpatchedSourceDefinition object + parsed_source.relation_name = self._get_relation_name(parsed_source) + return parsed_source + def create_test_node( self, target: Union[UnpatchedSourceDefinition, UnparsedNodeUpdate], diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index f4dd6917579..922a1fdc707 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1089,11 +1089,12 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): ) quote_database = quote_schema = self.adapter_type != 'snowflake' - relation_name_format = ".".join(( - self._quote("{0}") if quote_database else '{0}', - self._quote("{1}") if quote_schema else '{1}', - self._quote("{2}") if quote_model else '{2}', - )) + relation_name_node_format = self._relation_name_format( + quote_database, quote_schema, quote_model + ) + relation_name_source_format = self._relation_name_format( + quote_database, quote_schema, quote_identifier=True + ) return { 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', @@ -1103,7 +1104,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'build_path': Normalized('target/compiled/test/models/model.sql'), 'name': 'model', 'root_path': self.test_root_realpath, - 'relation_name': relation_name_format.format( + 'relation_name': relation_name_node_format.format( model_database, my_schema_name, 'model' ), 'resource_type': 'model', @@ -1179,7 +1180,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'build_path': Normalized('target/compiled/test/models/second_model.sql'), 'name': 'second_model', 'root_path': self.test_root_realpath, - 'relation_name': relation_name_format.format( + 'relation_name': relation_name_node_format.format( self.default_database, self.alternate_schema, 'second_model' ), @@ -1261,7 +1262,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_realpath, - 'relation_name': relation_name_format.format( + 'relation_name': relation_name_node_format.format( self.default_database, my_schema_name, 'seed' ), 'resource_type': 'seed', @@ -1405,8 +1406,9 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): .replace('{% snapshot snapshot_seed %}', '') .replace('{% endsnapshot %}', '')), 'refs': [['seed']], - 'relation_name': relation_name_format.format( - self.default_database, self.alternate_schema, 'snapshot_seed' + 'relation_name': relation_name_node_format.format( + self.default_database, self.alternate_schema, + 'snapshot_seed' ), 'resource_type': 'snapshot', 'root_path': self.test_root_realpath, @@ -1540,6 +1542,9 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'package_name': 'test', 'path': self.dir('models/schema.yml'), 'patch_path': None, + 'relation_name': relation_name_source_format.format( + self.default_database, my_schema_name, 'seed' + ), 'resource_type': 'source', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -1967,6 +1972,9 @@ def expected_postgres_references_manifest(self, model_database=None): 'package_name': 'test', 'path': self.dir('ref_models/schema.yml'), 'patch_path': None, + 'relation_name': '{0}."{1}"."seed"'.format( + self.default_database, my_schema_name + ), 'resource_type': 'source', 'root_path': self.test_root_realpath, 'schema': my_schema_name, @@ -2541,6 +2549,14 @@ def _absolute_path_to(self, searched_path: str, relative_path: str): normalize(relative_path) ) + def _relation_name_format(self, quote_database: bool, quote_schema: bool, + quote_identifier: bool): + return ".".join(( + self._quote("{0}") if quote_database else '{0}', + self._quote("{1}") if quote_schema else '{1}', + self._quote("{2}") if quote_identifier else '{2}', + )) + def expected_redshift_incremental_view_manifest(self): model_sql_path = self.dir('rs_models/model.sql') my_schema_name = self.unique_schema() @@ -2799,11 +2815,9 @@ def expected_run_results(self, quote_schema=True, quote_model=False, if quote_database else self.default_database) compiled_schema = self._quote(schema) if quote_schema else schema compiled_seed = self._quote('seed') if quote_model else 'seed' - relation_name_format = ".".join(( - self._quote("{0}") if quote_database else '{0}', - self._quote("{1}") if quote_schema else '{1}', - self._quote("{2}") if quote_model else '{2}', - )) + relation_name_format = self._relation_name_format( + quote_database, quote_schema, quote_model + ) if self.adapter_type == 'bigquery': compiled_sql = '\n\nselect * from `{}`.`{}`.seed'.format( From 3e5d9010a390de9b19bcae4263e3e90ed31a2a0a Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Wed, 28 Oct 2020 19:39:04 +0100 Subject: [PATCH 19/22] Add snapshot to additional Redshift and Bigquery manifest tests --- .../test_docs_generate.py | 85 ++++++++++++++++++- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 922a1fdc707..ca74f94c885 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -2129,6 +2129,7 @@ def expected_bigquery_complex_manifest(self): clustered_sql_path = self.dir('bq_models/clustered.sql') multi_clustered_sql_path = self.dir('bq_models/multi_clustered.sql') seed_path = self.dir('seed/seed.csv') + snapshot_path = self.dir('snapshot/snapshot_seed.sql') my_schema_name = self.unique_schema() return { @@ -2499,6 +2500,45 @@ def expected_bigquery_complex_manifest(self): 'checksum': self._checksum_file(seed_path), 'unrendered_config': self.unrendered_seed_config(), }, + 'snapshot.test.snapshot_seed': { + 'alias': 'snapshot_seed', + 'build_path': None, + 'checksum': self._checksum_file(snapshot_path), + 'columns': {}, + 'compiled': True, + 'compiled_sql': ANY, + 'config': self.rendered_snapshot_config( + target_schema=self.alternate_schema + ), + 'database': self.default_database, + 'deferred': False, + 'depends_on': {'macros': [], + 'nodes': ['seed.test.seed']}, + 'description': '', + 'docs': {'show': True}, + 'extra_ctes': [], + 'extra_ctes_injected': True, + 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], + 'meta': {}, + 'name': 'snapshot_seed', + 'original_file_path': snapshot_path, + 'package_name': 'test', + 'patch_path': None, + 'path': 'snapshot_seed.sql', + 'raw_sql': ANY, + 'refs': [['seed']], + 'relation_name': '`{0}`.`{1}`.snapshot_seed'.format( + self.default_database, self.alternate_schema + ), + 'resource_type': 'snapshot', + 'root_path': self.test_root_realpath, + 'schema': self.alternate_schema, + 'sources': [], + 'tags': [], + 'unique_id': 'snapshot.test.snapshot_seed', + 'unrendered_config': self.unrendered_snapshot_config( + target_schema=self.alternate_schema + )} }, 'sources': {}, 'exposures': {}, @@ -2561,6 +2601,7 @@ def expected_redshift_incremental_view_manifest(self): model_sql_path = self.dir('rs_models/model.sql') my_schema_name = self.unique_schema() seed_path = self.dir('seed/seed.csv') + snapshot_path = self.dir('snapshot/snapshot_seed.sql') return { 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', @@ -2724,6 +2765,45 @@ def expected_redshift_incremental_view_manifest(self): 'checksum': self._checksum_file(seed_path), 'unrendered_config': self.unrendered_seed_config(), }, + 'snapshot.test.snapshot_seed': { + 'alias': 'snapshot_seed', + 'build_path': None, + 'checksum': self._checksum_file(snapshot_path), + 'columns': {}, + 'compiled': True, + 'compiled_sql': ANY, + 'config': self.rendered_snapshot_config( + target_schema=self.alternate_schema + ), + 'database': self.default_database, + 'deferred': False, + 'depends_on': {'macros': [], + 'nodes': ['seed.test.seed']}, + 'description': '', + 'docs': {'show': True}, + 'extra_ctes': [], + 'extra_ctes_injected': True, + 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], + 'meta': {}, + 'name': 'snapshot_seed', + 'original_file_path': snapshot_path, + 'package_name': 'test', + 'patch_path': None, + 'path': 'snapshot_seed.sql', + 'raw_sql': ANY, + 'refs': [['seed']], + 'relation_name': '"{0}"."{1}".snapshot_seed'.format( + self.default_database, self.alternate_schema + ), + 'resource_type': 'snapshot', + 'root_path': self.test_root_realpath, + 'schema': self.alternate_schema, + 'sources': [], + 'tags': [], + 'unique_id': 'snapshot.test.snapshot_seed', + 'unrendered_config': self.unrendered_snapshot_config( + target_schema=self.alternate_schema + )} }, 'sources': {}, 'exposures': {}, @@ -3745,8 +3825,7 @@ def test__bigquery__run_and_generate(self): @use_profile('bigquery') def test__bigquery__complex_models(self): self.run_and_generate( - extra={'source-paths': [self.dir('bq_models')], - 'snapshot-paths': []}, + extra={'source-paths': [self.dir('bq_models')]}, model_count=4 ) @@ -3767,7 +3846,7 @@ def test__redshift__run_and_generate(self): @use_profile('redshift') def test__redshift__incremental_view(self): self.run_and_generate( - {'source-paths': [self.dir('rs_models')], 'snapshot-paths': []}, + {'source-paths': [self.dir('rs_models')]}, alternate_db=self.default_database, model_count=1, ) From 21fd75b5007b4b7d0a9d8120a356665f17ca2276 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Wed, 28 Oct 2020 19:59:36 +0100 Subject: [PATCH 20/22] Fix parent_map object in tests --- .../029_docs_generate_tests/test_docs_generate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index ca74f94c885..41559cf56fd 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -2553,6 +2553,7 @@ def expected_bigquery_complex_manifest(self): 'model.test.clustered': ['seed.test.seed'], 'model.test.multi_clustered': ['seed.test.seed'], 'seed.test.seed': [], + 'snapshot.test.snapshot_seed': ['seed.test.seed'], 'model.test.nested_table': [], 'model.test.nested_view': ['model.test.nested_table'], }, @@ -2813,7 +2814,9 @@ def expected_redshift_incremental_view_manifest(self): }, 'child_map': { 'model.test.model': [], - 'seed.test.seed': ['model.test.model'] + 'seed.test.seed': ['model.test.model', + 'snapshot.test.snapshot_seed'], + 'snapshot.test.snapshot_seed': [] }, 'docs': { 'dbt.__overview__': ANY, From 852990e96734911b61a047c3776e8a4c8f9fd193 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Wed, 28 Oct 2020 22:18:32 +0100 Subject: [PATCH 21/22] Fix child_map in tests --- .../029_docs_generate_tests/test_docs_generate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 41559cf56fd..b8fabe8059b 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -2547,7 +2547,9 @@ def expected_bigquery_complex_manifest(self): 'model.test.multi_clustered': [], 'model.test.nested_table': ['model.test.nested_view'], 'model.test.nested_view': [], - 'seed.test.seed': ['model.test.clustered', 'model.test.multi_clustered'] + 'seed.test.seed': ['model.test.clustered', + 'model.test.multi_clustered', + 'snapshot.test.snapshot_seed'] }, 'parent_map': { 'model.test.clustered': ['seed.test.seed'], @@ -2810,7 +2812,8 @@ def expected_redshift_incremental_view_manifest(self): 'exposures': {}, 'parent_map': { 'model.test.model': ['seed.test.seed'], - 'seed.test.seed': [] + 'seed.test.seed': [], + 'snapshot.test.snapshot_seed': ['seed.test.seed'] }, 'child_map': { 'model.test.model': [], From b741679c9cf49c153f822850be0797b49ec018b2 Mon Sep 17 00:00:00 2001 From: Fran Lozano Date: Thu, 29 Oct 2020 17:25:18 +0100 Subject: [PATCH 22/22] Add missing key to child map in expected_bigquery_complex_manifest --- test/integration/029_docs_generate_tests/test_docs_generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index b8fabe8059b..9d84d2e0b4e 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -2549,7 +2549,8 @@ def expected_bigquery_complex_manifest(self): 'model.test.nested_view': [], 'seed.test.seed': ['model.test.clustered', 'model.test.multi_clustered', - 'snapshot.test.snapshot_seed'] + 'snapshot.test.snapshot_seed'], + 'snapshot.test.snapshot_seed': [] }, 'parent_map': { 'model.test.clustered': ['seed.test.seed'],