From f0b07d0774ae2ac5a11a200e5a90c82c5b48a9a6 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 17 Aug 2023 18:02:24 -0400 Subject: [PATCH 01/11] first draft of adding in table - materialized view swap --- core/dbt/adapters/base/relation.py | 4 + .../macros/adapters/drop_relation.sql | 44 ----------- .../macros/adapters/relation.sql | 12 --- .../relations/materialized_view/_replace.sql} | 5 ++ .../relations/materialized_view/alter.sql} | 0 .../relations/materialized_view/create.sql} | 5 +- .../relations/materialized_view/drop.sql | 13 +++ .../relations/materialized_view/refresh.sql} | 2 +- .../relations/materialized_view/rename.sql | 10 +++ .../macros/adapters/relations/table/drop.sql | 13 +++ .../macros/adapters/relations/view/drop.sql | 13 +++ .../macros/relations/backup.sql | 17 ++++ .../macros/relations/create.sql | 23 ++++++ .../macros/relations/deploy_stage.sql | 14 ++++ .../global_project/macros/relations/drop.sql | 42 ++++++++++ .../macros/relations/drop_backup.sql | 14 ++++ .../macros/relations/rename.sql | 36 +++++++++ .../macros/relations/replace.sql | 35 ++++++++ .../global_project/macros/relations/stage.sql | 17 ++++ .../materializations/materialized_view.sql | 79 ------------------- .../relations/materialized_view/_replace.sql | 16 ++++ .../relations/materialized_view/alter.sql | 43 ++++++++++ .../relations/materialized_view/create.sql | 8 ++ .../relations/materialized_view/describe.sql | 5 ++ .../relations/materialized_view/refresh.sql | 3 + 25 files changed, 335 insertions(+), 138 deletions(-) delete mode 100644 core/dbt/include/global_project/macros/adapters/drop_relation.sql rename core/dbt/include/global_project/macros/{materializations/models/materialized_view/replace_materialized_view.sql => adapters/relations/materialized_view/_replace.sql} (72%) rename core/dbt/include/global_project/macros/{materializations/models/materialized_view/alter_materialized_view.sql => adapters/relations/materialized_view/alter.sql} (100%) rename core/dbt/include/global_project/macros/{materializations/models/materialized_view/create_materialized_view.sql => adapters/relations/materialized_view/create.sql} (62%) create mode 100644 core/dbt/include/global_project/macros/adapters/relations/materialized_view/drop.sql rename core/dbt/include/global_project/macros/{materializations/models/materialized_view/refresh_materialized_view.sql => adapters/relations/materialized_view/refresh.sql} (69%) create mode 100644 core/dbt/include/global_project/macros/adapters/relations/materialized_view/rename.sql create mode 100644 core/dbt/include/global_project/macros/adapters/relations/table/drop.sql create mode 100644 core/dbt/include/global_project/macros/adapters/relations/view/drop.sql create mode 100644 core/dbt/include/global_project/macros/relations/backup.sql create mode 100644 core/dbt/include/global_project/macros/relations/create.sql create mode 100644 core/dbt/include/global_project/macros/relations/deploy_stage.sql create mode 100644 core/dbt/include/global_project/macros/relations/drop.sql create mode 100644 core/dbt/include/global_project/macros/relations/drop_backup.sql create mode 100644 core/dbt/include/global_project/macros/relations/rename.sql create mode 100644 core/dbt/include/global_project/macros/relations/replace.sql create mode 100644 core/dbt/include/global_project/macros/relations/stage.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/_replace.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/alter.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/describe.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/refresh.sql diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index ae4e585d524..c5c44a8aca9 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -286,6 +286,10 @@ def create( ) return cls.from_dict(kwargs) + @property + def can_be_renamed(self): + return any([self.is_view, self.is_table, self.is_materialized_view]) + def __repr__(self) -> str: return "<{} {}>".format(self.__class__.__name__, self.render()) diff --git a/core/dbt/include/global_project/macros/adapters/drop_relation.sql b/core/dbt/include/global_project/macros/adapters/drop_relation.sql deleted file mode 100644 index bd254c78d51..00000000000 --- a/core/dbt/include/global_project/macros/adapters/drop_relation.sql +++ /dev/null @@ -1,44 +0,0 @@ -{% macro drop_relation(relation) -%} - {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }} -{% endmacro %} - -{% macro default__drop_relation(relation) -%} - {% call statement('drop_relation', auto_begin=False) -%} - {%- if relation.is_table -%} - {{- drop_table(relation) -}} - {%- elif relation.is_view -%} - {{- drop_view(relation) -}} - {%- elif relation.is_materialized_view -%} - {{- drop_materialized_view(relation) -}} - {%- else -%} - drop {{ relation.type }} if exists {{ relation }} cascade - {%- endif -%} - {%- endcall %} -{% endmacro %} - - -{% macro drop_table(relation) -%} - {{ return(adapter.dispatch('drop_table', 'dbt')(relation)) }} -{%- endmacro %} - -{% macro default__drop_table(relation) -%} - drop table if exists {{ relation }} cascade -{%- endmacro %} - - -{% macro drop_view(relation) -%} - {{ return(adapter.dispatch('drop_view', 'dbt')(relation)) }} -{%- endmacro %} - -{% macro default__drop_view(relation) -%} - drop view if exists {{ relation }} cascade -{%- endmacro %} - - -{% macro drop_materialized_view(relation) -%} - {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} -{%- endmacro %} - -{% macro default__drop_materialized_view(relation) -%} - drop materialized view if exists {{ relation }} cascade -{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relation.sql b/core/dbt/include/global_project/macros/adapters/relation.sql index f0dde7f20f0..228bb03dcc4 100644 --- a/core/dbt/include/global_project/macros/adapters/relation.sql +++ b/core/dbt/include/global_project/macros/adapters/relation.sql @@ -43,18 +43,6 @@ {% endmacro %} -{% macro rename_relation(from_relation, to_relation) -%} - {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }} -{% endmacro %} - -{% macro default__rename_relation(from_relation, to_relation) -%} - {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} - {% call statement('rename_relation') -%} - alter table {{ from_relation }} rename to {{ target_name }} - {%- endcall %} -{% endmacro %} - - {% macro get_or_create_relation(database, schema, identifier, type) -%} {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace_materialized_view.sql b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/_replace.sql similarity index 72% rename from core/dbt/include/global_project/macros/materializations/models/materialized_view/replace_materialized_view.sql rename to core/dbt/include/global_project/macros/adapters/relations/materialized_view/_replace.sql index 43319c5cc1b..5f227b864e2 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace_materialized_view.sql +++ b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/_replace.sql @@ -1,3 +1,8 @@ +{# /* +This only exists for backwards compatibility for 1.6.0. In later versions, the general `get_replace_sql` +macro is called as replace is inherently not limited to a single relation (it takes in two relations). +*/ #} + {% macro get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} {{- log('Applying REPLACE to: ' ~ relation) -}} {{- adapter.dispatch('get_replace_materialized_view_as_sql', 'dbt')(relation, sql, existing_relation, backup_relation, intermediate_relation) -}} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter_materialized_view.sql b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/alter.sql similarity index 100% rename from core/dbt/include/global_project/macros/materializations/models/materialized_view/alter_materialized_view.sql rename to core/dbt/include/global_project/macros/adapters/relations/materialized_view/alter.sql diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create_materialized_view.sql b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/create.sql similarity index 62% rename from core/dbt/include/global_project/macros/materializations/models/materialized_view/create_materialized_view.sql rename to core/dbt/include/global_project/macros/adapters/relations/materialized_view/create.sql index 4b2ebeb3aa1..a0f2614c9ca 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create_materialized_view.sql +++ b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/create.sql @@ -1,9 +1,10 @@ {% macro get_create_materialized_view_as_sql(relation, sql) -%} - {{- log('Applying CREATE to: ' ~ relation) -}} {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}} {%- endmacro %} {% macro default__get_create_materialized_view_as_sql(relation, sql) -%} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} + {{ exceptions.raise_compiler_error( + "`get_create_materialized_view_as_sql` has not been implemented for this adapter." + ) }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relations/materialized_view/drop.sql b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/drop.sql new file mode 100644 index 00000000000..d176d2b759d --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/drop.sql @@ -0,0 +1,13 @@ +{# /* +This was already implemented. Instead of creating a new macro that aligns with the standard, +this was reused and the default was maintained. This gets called by `drop_relation`, which +actually executes the drop, and `get_drop_sql`, which returns the template. +*/ #} + +{% macro drop_materialized_view(relation) -%} + {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} +{%- endmacro %} + +{% macro default__drop_materialized_view(relation) -%} + drop materialized view if exists {{ relation }} cascade +{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh_materialized_view.sql b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/refresh.sql similarity index 69% rename from core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh_materialized_view.sql rename to core/dbt/include/global_project/macros/adapters/relations/materialized_view/refresh.sql index 16345138593..d6b2732107a 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh_materialized_view.sql +++ b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/refresh.sql @@ -5,5 +5,5 @@ {% macro default__refresh_materialized_view(relation) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} + {{ exceptions.raise_compiler_error("`refresh_materialized_view` has not been implemented for this adapter.") }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relations/materialized_view/rename.sql b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/rename.sql new file mode 100644 index 00000000000..abd5babf68e --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/rename.sql @@ -0,0 +1,10 @@ +{% macro get_rename_materialized_view_sql(relation, new_name) %} + {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}} +{% endmacro %} + + +{% macro default__get_rename_materialized_view_sql(relation, new_name) %} + {{ exceptions.raise_compiler_error( + "`get_rename_materialized_view_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relations/table/drop.sql b/core/dbt/include/global_project/macros/adapters/relations/table/drop.sql new file mode 100644 index 00000000000..152676e3caa --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/relations/table/drop.sql @@ -0,0 +1,13 @@ +{# /* +This was already implemented. Instead of creating a new macro that aligns with the standard, +this was reused and the default was maintained. This gets called by `drop_relation`, which +actually executes the drop, and `get_drop_sql`, which returns the template. +*/ #} + +{% macro drop_table(relation) -%} + {{ return(adapter.dispatch('drop_table', 'dbt')(relation)) }} +{%- endmacro %} + +{% macro default__drop_table(relation) -%} + drop table if exists {{ relation }} cascade +{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relations/view/drop.sql b/core/dbt/include/global_project/macros/adapters/relations/view/drop.sql new file mode 100644 index 00000000000..46691b9e429 --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/relations/view/drop.sql @@ -0,0 +1,13 @@ +{# /* +This was already implemented. Instead of creating a new macro that aligns with the standard, +this was reused and the default was maintained. This gets called by `drop_relation`, which +actually executes the drop, and `get_drop_sql`, which returns the template. +*/ #} + +{% macro drop_view(relation) -%} + {{ return(adapter.dispatch('drop_view', 'dbt')(relation)) }} +{%- endmacro %} + +{% macro default__drop_view(relation) -%} + drop view if exists {{ relation }} cascade +{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/backup.sql b/core/dbt/include/global_project/macros/relations/backup.sql new file mode 100644 index 00000000000..2a5c7204321 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/backup.sql @@ -0,0 +1,17 @@ +{%- macro get_backup_sql(relation) -%} + {{- log('Applying BACKUP to: ' ~ relation) -}} + {{- adapter.dispatch('get_backup_sql', 'dbt')(relation) -}} +{%- endmacro -%} + + +{%- macro default__get_backup_sql(relation) -%} + + -- get the standard backup name + {% set backup_relation = make_backup_relation(relation, relation.type) %} + + -- drop any pre-existing backup + {{ get_drop_sql(backup_relation) }}; + + {{ get_rename_sql(relation, backup_relation) }} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/create.sql b/core/dbt/include/global_project/macros/relations/create.sql new file mode 100644 index 00000000000..3522392d2cb --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/create.sql @@ -0,0 +1,23 @@ +{%- macro get_create_sql(relation, sql) -%} + {{- log('Applying CREATE to: ' ~ relation) -}} + {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}} +{%- endmacro -%} + + +{%- macro default__get_create_sql(relation, sql) -%} + + {%- if relation.is_view -%} + {{ get_create_view_as_sql(relation, sql) }} + + {%- elif relation.is_table -%} + {{ get_create_table_as_sql(False, relation, sql) }} + + {%- elif relation.is_materialized_view -%} + {{ get_create_materialized_view_as_sql(relation, sql) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`get_create_sql` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/deploy_stage.sql b/core/dbt/include/global_project/macros/relations/deploy_stage.sql new file mode 100644 index 00000000000..93b383c268c --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/deploy_stage.sql @@ -0,0 +1,14 @@ +{%- macro get_deploy_stage_sql(relation) -%} + {{- log('Applying DEPLOY STAGE to: ' ~ relation) -}} + {{- adapter.dispatch('get_deploy_stage_sql', 'dbt')(relation) -}} +{%- endmacro -%} + + +{%- macro default__get_deploy_stage_sql(relation) -%} + + -- get the standard intermediate name + {% set intermediate_relation = make_intermediate_relation(relation) %} + + {{ get_rename_sql(intermediate_relation, relation) }} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/drop.sql b/core/dbt/include/global_project/macros/relations/drop.sql new file mode 100644 index 00000000000..0ddeee177e9 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/drop.sql @@ -0,0 +1,42 @@ +{%- macro get_drop_sql(relation) -%} + {{- log('Applying DROP to: ' ~ relation) -}} + {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}} +{%- endmacro -%} + + +{%- macro default__get_drop_sql(relation) -%} + + {%- if relation.is_view -%} + {{ drop_view(relation) }} + + {%- elif relation.is_table -%} + {{ drop_table(relation) }} + + {%- elif relation.is_materialized_view -%} + {{ drop_materialized_view(relation) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`get_drop_sql` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} + + +{% macro drop_relation(relation) -%} + {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }} +{% endmacro %} + +{% macro default__drop_relation(relation) -%} + {% call statement('drop_relation', auto_begin=False) -%} + {%- if relation.is_table -%} + {{- drop_table(relation) -}} + {%- elif relation.is_view -%} + {{- drop_view(relation) -}} + {%- elif relation.is_materialized_view -%} + {{- drop_materialized_view(relation) -}} + {%- else -%} + drop {{ relation.type }} if exists {{ relation }} cascade + {%- endif -%} + {%- endcall %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/drop_backup.sql b/core/dbt/include/global_project/macros/relations/drop_backup.sql new file mode 100644 index 00000000000..e70f7981e77 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/drop_backup.sql @@ -0,0 +1,14 @@ +{%- macro get_drop_backup_sql(relation) -%} + {{- log('Applying DROP BACKUP to: ' ~ relation) -}} + {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}} +{%- endmacro -%} + + +{%- macro default__get_drop_backup_sql(relation) -%} + + -- get the standard backup name + {% set backup_relation = make_backup_relation(relation, relation.type) %} + + {{ get_drop_sql(backup_relation) }} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/rename.sql b/core/dbt/include/global_project/macros/relations/rename.sql new file mode 100644 index 00000000000..04c7d8d7324 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/rename.sql @@ -0,0 +1,36 @@ +{%- macro get_rename_sql(relation, new_name) -%} + {{- log('Applying RENAME to: ' ~ relation) -}} + {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}} +{%- endmacro -%} + + +{%- macro default__get_rename_sql(relation, new_name) -%} + {% set target_name = adapter.quote_as_configured(new_name, 'identifier') %} + + {%- if relation.is_view -%} + {{ get_rename_view_sql(relation, target_name) }} + + {%- elif relation.is_table -%} + {{ get_rename_table_sql(relation, target_name) }} + + {%- elif relation.is_materialized_view -%} + {{ get_rename_materialized_view_sql(relation, target_name) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`get_rename_sql` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} + + +{% macro rename_relation(from_relation, to_relation) -%} + {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }} +{% endmacro %} + +{% macro default__rename_relation(from_relation, to_relation) -%} + {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} + {% call statement('rename_relation') -%} + alter table {{ from_relation }} rename to {{ target_name }} + {%- endcall %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/replace.sql b/core/dbt/include/global_project/macros/relations/replace.sql new file mode 100644 index 00000000000..888fe3bef93 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/replace.sql @@ -0,0 +1,35 @@ +{% macro get_replace_sql(existing_relation, target_relation, sql) %} + {{- log('Applying REPLACE to: ' ~ existing_relation) -}} + {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}} +{% endmacro %} + + +{% macro default__get_replace_sql(existing_relation, target_relation, sql) %} + + {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #} + {%- if target_relation.can_be_renamed and existing_relation.can_be_renamed -%} + {{ get_stage_sql(target_relation, sql) }}; + {{ get_backup_sql(existing_relation) }}; + {{ get_deploy_stage_sql(target_relation) }}; + {{ get_drop_backup_sql(existing_relation) }} + + {# /* create target_relation as an intermediate relation, then swap it out with the existing one using drop */ #} + {%- elif target_relation.can_be_renamed -%} + {{ get_stage_sql(target_relation, sql) }}; + {{ get_drop_sql(existing_relation) }}; + {{ get_deploy_stage_sql(target_relation) }} + + {# /* create target_relation in place by first backing up the existing relation */ #} + {%- elif existing_relation.can_be_renamed -%} + {{ get_backup_sql(existing_relation) }}; + {{ get_create_sql(target_relation, sql) }}; + {{ get_drop_backup_sql(existing_relation) }} + + {# /* no renaming is allowed, so just drop and create */ #} + {%- else -%} + {{ get_drop_sql(existing_relation) }}; + {{ get_create_sql(target_relation, sql) }} + + {%- endif -%} + +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/stage.sql b/core/dbt/include/global_project/macros/relations/stage.sql new file mode 100644 index 00000000000..08f56fa8f5d --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/stage.sql @@ -0,0 +1,17 @@ +{%- macro get_stage_sql(relation, sql) -%} + {{- log('Applying STAGE to: ' ~ relation) -}} + {{- adapter.dispatch('get_stage_sql', 'dbt')(relation, sql) -}} +{%- endmacro -%} + + +{%- macro default__get_stage_sql(relation, sql) -%} + + -- get the standard intermediate name + {% set intermediate_relation = make_intermediate_relation(relation) %} + + -- drop any pre-existing intermediate + {{ get_drop_sql(intermediate_relation) }}; + + {{ get_create_sql(intermediate_relation) }} + +{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql index 6e66e4bcd2c..77457d1f7e9 100644 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql +++ b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql @@ -1,84 +1,5 @@ -{% macro postgres__get_alter_materialized_view_as_sql( - relation, - configuration_changes, - sql, - existing_relation, - backup_relation, - intermediate_relation -) %} - - -- apply a full refresh immediately if needed - {% if configuration_changes.requires_full_refresh %} - - {{ get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) }} - - -- otherwise apply individual changes as needed - {% else %} - - {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }} - - {%- endif -%} - -{% endmacro %} - - -{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %} - create materialized view if not exists {{ relation }} as {{ sql }}; - - {% for _index_dict in config.get('indexes', []) -%} - {{- get_create_index_sql(relation, _index_dict) -}} - {%- endfor -%} - -{% endmacro %} - - -{% macro postgres__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} - {{- get_create_materialized_view_as_sql(intermediate_relation, sql) -}} - - {% if existing_relation is not none %} - alter materialized view {{ existing_relation }} rename to {{ backup_relation.include(database=False, schema=False) }}; - {% endif %} - - alter materialized view {{ intermediate_relation }} rename to {{ relation.include(database=False, schema=False) }}; - -{% endmacro %} - - {% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %} {% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %} {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config) %} {% do return(_configuration_changes) %} {% endmacro %} - - -{% macro postgres__refresh_materialized_view(relation) %} - refresh materialized view {{ relation }} -{% endmacro %} - - -{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%} - {{- log("Applying UPDATE INDEXES to: " ~ relation) -}} - - {%- for _index_change in index_changes -%} - {%- set _index = _index_change.context -%} - - {%- if _index_change.action == "drop" -%} - - {{ postgres__get_drop_index_sql(relation, _index.name) }}; - - {%- elif _index_change.action == "create" -%} - - {{ postgres__get_create_index_sql(relation, _index.as_node_config) }} - - {%- endif -%} - - {%- endfor -%} - -{%- endmacro -%} - - -{% macro postgres__describe_materialized_view(relation) %} - -- for now just get the indexes, we don't need the name or the query yet - {% set _indexes = run_query(get_show_indexes_sql(relation)) %} - {% do return({'indexes': _indexes}) %} -{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/_replace.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/_replace.sql new file mode 100644 index 00000000000..8e536756604 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/_replace.sql @@ -0,0 +1,16 @@ +{# /* +This only exists for backwards compatibility for 1.6.0. In later versions, the general `get_replace_sql` +macro is called as replace is inherently not limited to a single relation (it takes in two relations). +*/ #} + + +{% macro postgres__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} + {{- get_create_materialized_view_as_sql(intermediate_relation, sql) -}} + + {% if existing_relation is not none %} + alter materialized view {{ existing_relation }} rename to {{ backup_relation.include(database=False, schema=False) }}; + {% endif %} + + alter materialized view {{ intermediate_relation }} rename to {{ relation.include(database=False, schema=False) }}; + +{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/alter.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/alter.sql new file mode 100644 index 00000000000..d1270303bd7 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/alter.sql @@ -0,0 +1,43 @@ +{% macro postgres__get_alter_materialized_view_as_sql( + relation, + configuration_changes, + sql, + existing_relation, + backup_relation, + intermediate_relation +) %} + + -- apply a full refresh immediately if needed + {% if configuration_changes.requires_full_refresh %} + + {{ get_replace_sql(existing_relation, relation, sql) }} + + -- otherwise apply individual changes as needed + {% else %} + + {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }} + + {%- endif -%} + +{% endmacro %} + + +{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%} + {{- log("Applying UPDATE INDEXES to: " ~ relation) -}} + + {%- for _index_change in index_changes -%} + {%- set _index = _index_change.context -%} + + {%- if _index_change.action == "drop" -%} + + {{ postgres__get_drop_index_sql(relation, _index.name) }}; + + {%- elif _index_change.action == "create" -%} + + {{ postgres__get_create_index_sql(relation, _index.as_node_config) }} + + {%- endif -%} + + {%- endfor -%} + +{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql new file mode 100644 index 00000000000..17e5cb06434 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql @@ -0,0 +1,8 @@ +{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %} + create materialized view if not exists {{ relation }} as {{ sql }}; + + {% for _index_dict in config.get('indexes', []) -%} + {{- get_create_index_sql(relation, _index_dict) -}} + {%- endfor -%} + +{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/describe.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/describe.sql new file mode 100644 index 00000000000..cb133b6a8b5 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/describe.sql @@ -0,0 +1,5 @@ +{% macro postgres__describe_materialized_view(relation) %} + -- for now just get the indexes, we don't need the name or the query yet + {% set _indexes = run_query(get_show_indexes_sql(relation)) %} + {% do return({'indexes': _indexes}) %} +{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/refresh.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/refresh.sql new file mode 100644 index 00000000000..48b863e519b --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/refresh.sql @@ -0,0 +1,3 @@ +{% macro postgres__refresh_materialized_view(relation) %} + refresh materialized view {{ relation }} +{% endmacro %} From 73724407cbce62cede267fde118f8df224f43a40 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 17 Aug 2023 18:09:46 -0400 Subject: [PATCH 02/11] tests pass --- .../test_materialized_view.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py index 5f50afc994c..a491c781d9b 100644 --- a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py @@ -56,18 +56,6 @@ def query_row_count(project, relation: BaseRelation) -> int: def query_relation_type(project, relation: BaseRelation) -> Optional[str]: return query_relation_type(project, relation) - @pytest.mark.skip( - "The current implementation does not support overwriting materialized views with tables." - ) - def test_table_replaces_materialized_view(self, project, my_materialized_view): - super().test_table_replaces_materialized_view(project, my_materialized_view) - - @pytest.mark.skip( - "The current implementation does not support overwriting materialized views with views." - ) - def test_view_replaces_materialized_view(self, project, my_materialized_view): - super().test_view_replaces_materialized_view(project, my_materialized_view) - @pytest.mark.skip( "The current implementation does not support overwriting tables with materialized views." ) From d20f7dc7c7d101121cf34edd70e4db71f04b3449 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 17 Aug 2023 18:55:52 -0400 Subject: [PATCH 03/11] table/view/materialized view can all replace each other --- .../macros/adapters/relations/table/rename.sql | 10 ++++++++++ .../macros/adapters/relations/view/rename.sql | 10 ++++++++++ .../models/materialized_view/materialized_view.sql | 2 +- .../global_project/macros/relations/backup.sql | 2 +- .../global_project/macros/relations/deploy_stage.sql | 2 +- .../global_project/macros/relations/rename.sql | 7 +++---- .../global_project/macros/relations/stage.sql | 2 +- .../macros/relations/materialized_view/rename.sql | 3 +++ .../postgres/macros/relations/table/rename.sql | 3 +++ .../postgres/macros/relations/view/rename.sql | 3 +++ .../test_materialized_view.py | 12 ------------ 11 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 core/dbt/include/global_project/macros/adapters/relations/table/rename.sql create mode 100644 core/dbt/include/global_project/macros/adapters/relations/view/rename.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/table/rename.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/view/rename.sql diff --git a/core/dbt/include/global_project/macros/adapters/relations/table/rename.sql b/core/dbt/include/global_project/macros/adapters/relations/table/rename.sql new file mode 100644 index 00000000000..4688780d68d --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/relations/table/rename.sql @@ -0,0 +1,10 @@ +{% macro get_rename_table_sql(relation, new_name) %} + {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}} +{% endmacro %} + + +{% macro default__get_rename_table_sql(relation, new_name) %} + {{ exceptions.raise_compiler_error( + "`get_rename_table_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relations/view/rename.sql b/core/dbt/include/global_project/macros/adapters/relations/view/rename.sql new file mode 100644 index 00000000000..165dd514e40 --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/relations/view/rename.sql @@ -0,0 +1,10 @@ +{% macro get_rename_view_sql(relation, new_name) %} + {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}} +{% endmacro %} + + +{% macro default__get_rename_view_sql(relation, new_name) %} + {{ exceptions.raise_compiler_error( + "`get_rename_view_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql index 015f6cb8585..6dc30bf9a9a 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql @@ -57,7 +57,7 @@ {% if existing_relation is none %} {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %} {% elif full_refresh_mode or not existing_relation.is_materialized_view %} - {% set build_sql = get_replace_materialized_view_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) %} + {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %} {% else %} -- get config options diff --git a/core/dbt/include/global_project/macros/relations/backup.sql b/core/dbt/include/global_project/macros/relations/backup.sql index 2a5c7204321..928af861b5a 100644 --- a/core/dbt/include/global_project/macros/relations/backup.sql +++ b/core/dbt/include/global_project/macros/relations/backup.sql @@ -12,6 +12,6 @@ -- drop any pre-existing backup {{ get_drop_sql(backup_relation) }}; - {{ get_rename_sql(relation, backup_relation) }} + {{ get_rename_sql(relation, backup_relation.identifier) }} {%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/deploy_stage.sql b/core/dbt/include/global_project/macros/relations/deploy_stage.sql index 93b383c268c..909e06332ee 100644 --- a/core/dbt/include/global_project/macros/relations/deploy_stage.sql +++ b/core/dbt/include/global_project/macros/relations/deploy_stage.sql @@ -9,6 +9,6 @@ -- get the standard intermediate name {% set intermediate_relation = make_intermediate_relation(relation) %} - {{ get_rename_sql(intermediate_relation, relation) }} + {{ get_rename_sql(intermediate_relation, relation.identifier) }} {%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/rename.sql b/core/dbt/include/global_project/macros/relations/rename.sql index 04c7d8d7324..d7f3a72e29a 100644 --- a/core/dbt/include/global_project/macros/relations/rename.sql +++ b/core/dbt/include/global_project/macros/relations/rename.sql @@ -5,16 +5,15 @@ {%- macro default__get_rename_sql(relation, new_name) -%} - {% set target_name = adapter.quote_as_configured(new_name, 'identifier') %} {%- if relation.is_view -%} - {{ get_rename_view_sql(relation, target_name) }} + {{ get_rename_view_sql(relation, new_name) }} {%- elif relation.is_table -%} - {{ get_rename_table_sql(relation, target_name) }} + {{ get_rename_table_sql(relation, new_name) }} {%- elif relation.is_materialized_view -%} - {{ get_rename_materialized_view_sql(relation, target_name) }} + {{ get_rename_materialized_view_sql(relation, new_name) }} {%- else -%} {{- exceptions.raise_compiler_error("`get_rename_sql` has not been implemented for: " ~ relation.type ) -}} diff --git a/core/dbt/include/global_project/macros/relations/stage.sql b/core/dbt/include/global_project/macros/relations/stage.sql index 08f56fa8f5d..d52df2d4a05 100644 --- a/core/dbt/include/global_project/macros/relations/stage.sql +++ b/core/dbt/include/global_project/macros/relations/stage.sql @@ -12,6 +12,6 @@ -- drop any pre-existing intermediate {{ get_drop_sql(intermediate_relation) }}; - {{ get_create_sql(intermediate_relation) }} + {{ get_create_sql(intermediate_relation, sql) }} {%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql new file mode 100644 index 00000000000..293ec9d1e12 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql @@ -0,0 +1,3 @@ +{% macro postgres__get_rename_materialized_view_sql(relation, new_name) %} + alter materialized view {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/table/rename.sql b/plugins/postgres/dbt/include/postgres/macros/relations/table/rename.sql new file mode 100644 index 00000000000..bc3c234abb7 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/table/rename.sql @@ -0,0 +1,3 @@ +{% macro postgres__get_rename_table_sql(relation, new_name) %} + alter table {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/view/rename.sql b/plugins/postgres/dbt/include/postgres/macros/relations/view/rename.sql new file mode 100644 index 00000000000..3c890a5b2d0 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/view/rename.sql @@ -0,0 +1,3 @@ +{% macro postgres__get_rename_view_sql(relation, new_name) %} + alter view {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py index a491c781d9b..136186a5ec6 100644 --- a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py @@ -56,18 +56,6 @@ def query_row_count(project, relation: BaseRelation) -> int: def query_relation_type(project, relation: BaseRelation) -> Optional[str]: return query_relation_type(project, relation) - @pytest.mark.skip( - "The current implementation does not support overwriting tables with materialized views." - ) - def test_materialized_view_replaces_table(self, project, my_table): - super().test_materialized_view_replaces_table(project, my_table) - - @pytest.mark.skip( - "The current implementation does not support overwriting views with materialized views." - ) - def test_materialized_view_replaces_view(self, project, my_view): - super().test_materialized_view_replaces_view(project, my_view) - class PostgresMaterializedViewChanges(MaterializedViewChanges): @pytest.fixture(scope="class", autouse=True) From 28e161270029e5a623213bea5d68880ebc7d065f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 17 Aug 2023 18:57:55 -0400 Subject: [PATCH 04/11] changie --- .changes/unreleased/Fixes-20230817-185739.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20230817-185739.yaml diff --git a/.changes/unreleased/Fixes-20230817-185739.yaml b/.changes/unreleased/Fixes-20230817-185739.yaml new file mode 100644 index 00000000000..74b1ed7551e --- /dev/null +++ b/.changes/unreleased/Fixes-20230817-185739.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Add support for swapping materialized views with tables/views and vice versa +time: 2023-08-17T18:57:39.01958-04:00 +custom: + Author: mikealfare + Issue: "8449" From 4483f6ca78e7c5d2b78fcfe05a501cc8046a147b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 17 Aug 2023 19:35:33 -0400 Subject: [PATCH 05/11] update renameable relations to a config --- core/dbt/adapters/base/relation.py | 6 +++--- .../dbt/adapters/postgres/relation.py | 7 +++++++ tests/unit/test_relation.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_relation.py diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index c5c44a8aca9..c22dba56166 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -1,6 +1,6 @@ from collections.abc import Hashable from dataclasses import dataclass, field -from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set +from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set, List from dbt.contracts.graph.nodes import SourceDefinition, ManifestNode, ResultNode, ParsedNode from dbt.contracts.relation import ( @@ -35,6 +35,7 @@ class BaseRelation(FakeAPIObject, Hashable): include_policy: Policy = field(default_factory=lambda: Policy()) quote_policy: Policy = field(default_factory=lambda: Policy()) dbt_created: bool = False + relations_that_can_be_renamed: List[RelationType] = field(default_factory=list) def _is_exactish_match(self, field: ComponentName, value: str) -> bool: if self.dbt_created and self.quote_policy.get_part(field) is False: @@ -169,7 +170,6 @@ def without_identifier(self) -> "BaseRelation": return self.include(identifier=False).replace_path(identifier=None) def _render_iterator(self) -> Iterator[Tuple[Optional[ComponentName], Optional[str]]]: - for key in ComponentName: path_part: Optional[str] = None if self.include_policy.get_part(key): @@ -288,7 +288,7 @@ def create( @property def can_be_renamed(self): - return any([self.is_view, self.is_table, self.is_materialized_view]) + return self.type in self.relations_that_can_be_renamed def __repr__(self) -> str: return "<{} {}>".format(self.__class__.__name__, self.render()) diff --git a/plugins/postgres/dbt/adapters/postgres/relation.py b/plugins/postgres/dbt/adapters/postgres/relation.py index 43822efb11f..26516af4819 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation.py +++ b/plugins/postgres/dbt/adapters/postgres/relation.py @@ -7,6 +7,7 @@ RelationResults, ) from dbt.context.providers import RuntimeConfigObject +from dbt.contracts.relation import RelationType from dbt.exceptions import DbtRuntimeError from dbt.adapters.postgres.relation_configs import ( @@ -20,6 +21,12 @@ @dataclass(frozen=True, eq=False, repr=False) class PostgresRelation(BaseRelation): + relations_that_can_be_renamed = [ + RelationType.View, + RelationType.Table, + RelationType.MaterializedView, + ] + def __post_init__(self): # Check for length of Postgres table/view names. # Check self.type to exclude test relation identifiers diff --git a/tests/unit/test_relation.py b/tests/unit/test_relation.py new file mode 100644 index 00000000000..3e0ebf7fac8 --- /dev/null +++ b/tests/unit/test_relation.py @@ -0,0 +1,19 @@ +from dataclasses import replace + +import pytest + +from dbt.adapters.base import BaseRelation +from dbt.contracts.relation import RelationType + + +@pytest.mark.parametrize( + "relation_type,result", + [ + (RelationType.View, True), + (RelationType.Table, False), + ], +) +def test_can_be_renamed(relation_type, result): + my_relation = BaseRelation.create(type=relation_type) + my_relation = replace(my_relation, relations_that_can_be_renamed=[RelationType.View]) + assert my_relation.can_be_renamed is result From 03a15e7be0248338c439cd0f29a4d4fb2d969e55 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 18 Aug 2023 16:00:07 -0400 Subject: [PATCH 06/11] revert to just reformatting changes --- core/dbt/adapters/base/relation.py | 7 +--- .../relations/materialized_view/rename.sql | 10 ------ .../adapters/relations/table/rename.sql | 10 ------ .../macros/adapters/relations/view/rename.sql | 10 ------ .../materialized_view/materialized_view.sql | 2 +- .../macros/relations/backup.sql | 17 --------- .../macros/relations/create.sql | 23 ------------ .../macros/relations/deploy_stage.sql | 14 -------- .../global_project/macros/relations/drop.sql | 25 ------------- .../macros/relations/drop_backup.sql | 14 -------- .../macros/relations/rename.sql | 25 ------------- .../macros/relations/replace.sql | 35 ------------------- .../global_project/macros/relations/stage.sql | 17 --------- .../dbt/adapters/postgres/relation.py | 7 ---- .../relations/materialized_view/alter.sql | 2 +- .../macros/relations/table/rename.sql | 3 -- .../postgres/macros/relations/view/rename.sql | 3 -- .../test_materialized_view.py | 14 ++++++++ 18 files changed, 17 insertions(+), 221 deletions(-) delete mode 100644 core/dbt/include/global_project/macros/adapters/relations/materialized_view/rename.sql delete mode 100644 core/dbt/include/global_project/macros/adapters/relations/table/rename.sql delete mode 100644 core/dbt/include/global_project/macros/adapters/relations/view/rename.sql delete mode 100644 core/dbt/include/global_project/macros/relations/backup.sql delete mode 100644 core/dbt/include/global_project/macros/relations/create.sql delete mode 100644 core/dbt/include/global_project/macros/relations/deploy_stage.sql delete mode 100644 core/dbt/include/global_project/macros/relations/drop_backup.sql delete mode 100644 core/dbt/include/global_project/macros/relations/replace.sql delete mode 100644 core/dbt/include/global_project/macros/relations/stage.sql delete mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/table/rename.sql delete mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/view/rename.sql diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index c22dba56166..998d809d06e 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -1,6 +1,6 @@ from collections.abc import Hashable from dataclasses import dataclass, field -from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set, List +from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set from dbt.contracts.graph.nodes import SourceDefinition, ManifestNode, ResultNode, ParsedNode from dbt.contracts.relation import ( @@ -35,7 +35,6 @@ class BaseRelation(FakeAPIObject, Hashable): include_policy: Policy = field(default_factory=lambda: Policy()) quote_policy: Policy = field(default_factory=lambda: Policy()) dbt_created: bool = False - relations_that_can_be_renamed: List[RelationType] = field(default_factory=list) def _is_exactish_match(self, field: ComponentName, value: str) -> bool: if self.dbt_created and self.quote_policy.get_part(field) is False: @@ -286,10 +285,6 @@ def create( ) return cls.from_dict(kwargs) - @property - def can_be_renamed(self): - return self.type in self.relations_that_can_be_renamed - def __repr__(self) -> str: return "<{} {}>".format(self.__class__.__name__, self.render()) diff --git a/core/dbt/include/global_project/macros/adapters/relations/materialized_view/rename.sql b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/rename.sql deleted file mode 100644 index abd5babf68e..00000000000 --- a/core/dbt/include/global_project/macros/adapters/relations/materialized_view/rename.sql +++ /dev/null @@ -1,10 +0,0 @@ -{% macro get_rename_materialized_view_sql(relation, new_name) %} - {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}} -{% endmacro %} - - -{% macro default__get_rename_materialized_view_sql(relation, new_name) %} - {{ exceptions.raise_compiler_error( - "`get_rename_materialized_view_sql` has not been implemented for this adapter." - ) }} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relations/table/rename.sql b/core/dbt/include/global_project/macros/adapters/relations/table/rename.sql deleted file mode 100644 index 4688780d68d..00000000000 --- a/core/dbt/include/global_project/macros/adapters/relations/table/rename.sql +++ /dev/null @@ -1,10 +0,0 @@ -{% macro get_rename_table_sql(relation, new_name) %} - {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}} -{% endmacro %} - - -{% macro default__get_rename_table_sql(relation, new_name) %} - {{ exceptions.raise_compiler_error( - "`get_rename_table_sql` has not been implemented for this adapter." - ) }} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relations/view/rename.sql b/core/dbt/include/global_project/macros/adapters/relations/view/rename.sql deleted file mode 100644 index 165dd514e40..00000000000 --- a/core/dbt/include/global_project/macros/adapters/relations/view/rename.sql +++ /dev/null @@ -1,10 +0,0 @@ -{% macro get_rename_view_sql(relation, new_name) %} - {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}} -{% endmacro %} - - -{% macro default__get_rename_view_sql(relation, new_name) %} - {{ exceptions.raise_compiler_error( - "`get_rename_view_sql` has not been implemented for this adapter." - ) }} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql index 6dc30bf9a9a..015f6cb8585 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql @@ -57,7 +57,7 @@ {% if existing_relation is none %} {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %} {% elif full_refresh_mode or not existing_relation.is_materialized_view %} - {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %} + {% set build_sql = get_replace_materialized_view_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) %} {% else %} -- get config options diff --git a/core/dbt/include/global_project/macros/relations/backup.sql b/core/dbt/include/global_project/macros/relations/backup.sql deleted file mode 100644 index 928af861b5a..00000000000 --- a/core/dbt/include/global_project/macros/relations/backup.sql +++ /dev/null @@ -1,17 +0,0 @@ -{%- macro get_backup_sql(relation) -%} - {{- log('Applying BACKUP to: ' ~ relation) -}} - {{- adapter.dispatch('get_backup_sql', 'dbt')(relation) -}} -{%- endmacro -%} - - -{%- macro default__get_backup_sql(relation) -%} - - -- get the standard backup name - {% set backup_relation = make_backup_relation(relation, relation.type) %} - - -- drop any pre-existing backup - {{ get_drop_sql(backup_relation) }}; - - {{ get_rename_sql(relation, backup_relation.identifier) }} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/create.sql b/core/dbt/include/global_project/macros/relations/create.sql deleted file mode 100644 index 3522392d2cb..00000000000 --- a/core/dbt/include/global_project/macros/relations/create.sql +++ /dev/null @@ -1,23 +0,0 @@ -{%- macro get_create_sql(relation, sql) -%} - {{- log('Applying CREATE to: ' ~ relation) -}} - {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}} -{%- endmacro -%} - - -{%- macro default__get_create_sql(relation, sql) -%} - - {%- if relation.is_view -%} - {{ get_create_view_as_sql(relation, sql) }} - - {%- elif relation.is_table -%} - {{ get_create_table_as_sql(False, relation, sql) }} - - {%- elif relation.is_materialized_view -%} - {{ get_create_materialized_view_as_sql(relation, sql) }} - - {%- else -%} - {{- exceptions.raise_compiler_error("`get_create_sql` has not been implemented for: " ~ relation.type ) -}} - - {%- endif -%} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/deploy_stage.sql b/core/dbt/include/global_project/macros/relations/deploy_stage.sql deleted file mode 100644 index 909e06332ee..00000000000 --- a/core/dbt/include/global_project/macros/relations/deploy_stage.sql +++ /dev/null @@ -1,14 +0,0 @@ -{%- macro get_deploy_stage_sql(relation) -%} - {{- log('Applying DEPLOY STAGE to: ' ~ relation) -}} - {{- adapter.dispatch('get_deploy_stage_sql', 'dbt')(relation) -}} -{%- endmacro -%} - - -{%- macro default__get_deploy_stage_sql(relation) -%} - - -- get the standard intermediate name - {% set intermediate_relation = make_intermediate_relation(relation) %} - - {{ get_rename_sql(intermediate_relation, relation.identifier) }} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/drop.sql b/core/dbt/include/global_project/macros/relations/drop.sql index 0ddeee177e9..d02ad9327d2 100644 --- a/core/dbt/include/global_project/macros/relations/drop.sql +++ b/core/dbt/include/global_project/macros/relations/drop.sql @@ -1,28 +1,3 @@ -{%- macro get_drop_sql(relation) -%} - {{- log('Applying DROP to: ' ~ relation) -}} - {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}} -{%- endmacro -%} - - -{%- macro default__get_drop_sql(relation) -%} - - {%- if relation.is_view -%} - {{ drop_view(relation) }} - - {%- elif relation.is_table -%} - {{ drop_table(relation) }} - - {%- elif relation.is_materialized_view -%} - {{ drop_materialized_view(relation) }} - - {%- else -%} - {{- exceptions.raise_compiler_error("`get_drop_sql` has not been implemented for: " ~ relation.type ) -}} - - {%- endif -%} - -{%- endmacro -%} - - {% macro drop_relation(relation) -%} {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/drop_backup.sql b/core/dbt/include/global_project/macros/relations/drop_backup.sql deleted file mode 100644 index e70f7981e77..00000000000 --- a/core/dbt/include/global_project/macros/relations/drop_backup.sql +++ /dev/null @@ -1,14 +0,0 @@ -{%- macro get_drop_backup_sql(relation) -%} - {{- log('Applying DROP BACKUP to: ' ~ relation) -}} - {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}} -{%- endmacro -%} - - -{%- macro default__get_drop_backup_sql(relation) -%} - - -- get the standard backup name - {% set backup_relation = make_backup_relation(relation, relation.type) %} - - {{ get_drop_sql(backup_relation) }} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/rename.sql b/core/dbt/include/global_project/macros/relations/rename.sql index d7f3a72e29a..7d6c979cfc1 100644 --- a/core/dbt/include/global_project/macros/relations/rename.sql +++ b/core/dbt/include/global_project/macros/relations/rename.sql @@ -1,28 +1,3 @@ -{%- macro get_rename_sql(relation, new_name) -%} - {{- log('Applying RENAME to: ' ~ relation) -}} - {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}} -{%- endmacro -%} - - -{%- macro default__get_rename_sql(relation, new_name) -%} - - {%- if relation.is_view -%} - {{ get_rename_view_sql(relation, new_name) }} - - {%- elif relation.is_table -%} - {{ get_rename_table_sql(relation, new_name) }} - - {%- elif relation.is_materialized_view -%} - {{ get_rename_materialized_view_sql(relation, new_name) }} - - {%- else -%} - {{- exceptions.raise_compiler_error("`get_rename_sql` has not been implemented for: " ~ relation.type ) -}} - - {%- endif -%} - -{%- endmacro -%} - - {% macro rename_relation(from_relation, to_relation) -%} {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/replace.sql b/core/dbt/include/global_project/macros/relations/replace.sql deleted file mode 100644 index 888fe3bef93..00000000000 --- a/core/dbt/include/global_project/macros/relations/replace.sql +++ /dev/null @@ -1,35 +0,0 @@ -{% macro get_replace_sql(existing_relation, target_relation, sql) %} - {{- log('Applying REPLACE to: ' ~ existing_relation) -}} - {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}} -{% endmacro %} - - -{% macro default__get_replace_sql(existing_relation, target_relation, sql) %} - - {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #} - {%- if target_relation.can_be_renamed and existing_relation.can_be_renamed -%} - {{ get_stage_sql(target_relation, sql) }}; - {{ get_backup_sql(existing_relation) }}; - {{ get_deploy_stage_sql(target_relation) }}; - {{ get_drop_backup_sql(existing_relation) }} - - {# /* create target_relation as an intermediate relation, then swap it out with the existing one using drop */ #} - {%- elif target_relation.can_be_renamed -%} - {{ get_stage_sql(target_relation, sql) }}; - {{ get_drop_sql(existing_relation) }}; - {{ get_deploy_stage_sql(target_relation) }} - - {# /* create target_relation in place by first backing up the existing relation */ #} - {%- elif existing_relation.can_be_renamed -%} - {{ get_backup_sql(existing_relation) }}; - {{ get_create_sql(target_relation, sql) }}; - {{ get_drop_backup_sql(existing_relation) }} - - {# /* no renaming is allowed, so just drop and create */ #} - {%- else -%} - {{ get_drop_sql(existing_relation) }}; - {{ get_create_sql(target_relation, sql) }} - - {%- endif -%} - -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/stage.sql b/core/dbt/include/global_project/macros/relations/stage.sql deleted file mode 100644 index d52df2d4a05..00000000000 --- a/core/dbt/include/global_project/macros/relations/stage.sql +++ /dev/null @@ -1,17 +0,0 @@ -{%- macro get_stage_sql(relation, sql) -%} - {{- log('Applying STAGE to: ' ~ relation) -}} - {{- adapter.dispatch('get_stage_sql', 'dbt')(relation, sql) -}} -{%- endmacro -%} - - -{%- macro default__get_stage_sql(relation, sql) -%} - - -- get the standard intermediate name - {% set intermediate_relation = make_intermediate_relation(relation) %} - - -- drop any pre-existing intermediate - {{ get_drop_sql(intermediate_relation) }}; - - {{ get_create_sql(intermediate_relation, sql) }} - -{%- endmacro -%} diff --git a/plugins/postgres/dbt/adapters/postgres/relation.py b/plugins/postgres/dbt/adapters/postgres/relation.py index 26516af4819..43822efb11f 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation.py +++ b/plugins/postgres/dbt/adapters/postgres/relation.py @@ -7,7 +7,6 @@ RelationResults, ) from dbt.context.providers import RuntimeConfigObject -from dbt.contracts.relation import RelationType from dbt.exceptions import DbtRuntimeError from dbt.adapters.postgres.relation_configs import ( @@ -21,12 +20,6 @@ @dataclass(frozen=True, eq=False, repr=False) class PostgresRelation(BaseRelation): - relations_that_can_be_renamed = [ - RelationType.View, - RelationType.Table, - RelationType.MaterializedView, - ] - def __post_init__(self): # Check for length of Postgres table/view names. # Check self.type to exclude test relation identifiers diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/alter.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/alter.sql index d1270303bd7..0dd5189783d 100644 --- a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/alter.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/alter.sql @@ -10,7 +10,7 @@ -- apply a full refresh immediately if needed {% if configuration_changes.requires_full_refresh %} - {{ get_replace_sql(existing_relation, relation, sql) }} + {{ get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) }} -- otherwise apply individual changes as needed {% else %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/table/rename.sql b/plugins/postgres/dbt/include/postgres/macros/relations/table/rename.sql deleted file mode 100644 index bc3c234abb7..00000000000 --- a/plugins/postgres/dbt/include/postgres/macros/relations/table/rename.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro postgres__get_rename_table_sql(relation, new_name) %} - alter table {{ relation }} rename to {{ new_name }} -{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/view/rename.sql b/plugins/postgres/dbt/include/postgres/macros/relations/view/rename.sql deleted file mode 100644 index 3c890a5b2d0..00000000000 --- a/plugins/postgres/dbt/include/postgres/macros/relations/view/rename.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro postgres__get_rename_view_sql(relation, new_name) %} - alter view {{ relation }} rename to {{ new_name }} -{% endmacro %} diff --git a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py index 136186a5ec6..9d950356d0a 100644 --- a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py @@ -56,6 +56,20 @@ def query_row_count(project, relation: BaseRelation) -> int: def query_relation_type(project, relation: BaseRelation) -> Optional[str]: return query_relation_type(project, relation) + @pytest.mark.skip("") + def test_materialized_view_replaces_table(self, project, my_table): + super().test_materialized_view_replaces_table(project, my_table) + + @pytest.mark.skip("") + def test_materialized_view_replaces_view(self, project, my_view): + super().test_materialized_view_replaces_view(project, my_view) + + def test_table_replaces_materialized_view(self, project, my_materialized_view): + super().test_table_replaces_materialized_view(project, my_materialized_view) + + def test_view_replaces_materialized_view(self, project, my_materialized_view): + super().test_view_replaces_materialized_view(project, my_materialized_view) + class PostgresMaterializedViewChanges(MaterializedViewChanges): @pytest.fixture(scope="class", autouse=True) From d755c5159a05963f07eaad8d6703e09f8434bf8a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 18 Aug 2023 16:07:49 -0400 Subject: [PATCH 07/11] revert to just reformatting changes --- .changes/unreleased/Fixes-20230817-185739.yaml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .changes/unreleased/Fixes-20230817-185739.yaml diff --git a/.changes/unreleased/Fixes-20230817-185739.yaml b/.changes/unreleased/Fixes-20230817-185739.yaml deleted file mode 100644 index 74b1ed7551e..00000000000 --- a/.changes/unreleased/Fixes-20230817-185739.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Add support for swapping materialized views with tables/views and vice versa -time: 2023-08-17T18:57:39.01958-04:00 -custom: - Author: mikealfare - Issue: "8449" From d8984699c2a96d3622ee82853721fd84503f7c1a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 18 Aug 2023 16:08:52 -0400 Subject: [PATCH 08/11] revert to just reformatting changes --- core/dbt/adapters/base/relation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index 998d809d06e..ae4e585d524 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -169,6 +169,7 @@ def without_identifier(self) -> "BaseRelation": return self.include(identifier=False).replace_path(identifier=None) def _render_iterator(self) -> Iterator[Tuple[Optional[ComponentName], Optional[str]]]: + for key in ComponentName: path_part: Optional[str] = None if self.include_policy.get_part(key): From 46e8aef248518b039cf698fba268b3dc3de93c5e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 18 Aug 2023 21:48:32 -0400 Subject: [PATCH 09/11] revert to just reformatting changes --- .../test_materialized_view.py | 26 +++++++++++++------ tests/unit/test_relation.py | 19 -------------- 2 files changed, 18 insertions(+), 27 deletions(-) delete mode 100644 tests/unit/test_relation.py diff --git a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py index 9d950356d0a..5f50afc994c 100644 --- a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py @@ -56,20 +56,30 @@ def query_row_count(project, relation: BaseRelation) -> int: def query_relation_type(project, relation: BaseRelation) -> Optional[str]: return query_relation_type(project, relation) - @pytest.mark.skip("") - def test_materialized_view_replaces_table(self, project, my_table): - super().test_materialized_view_replaces_table(project, my_table) - - @pytest.mark.skip("") - def test_materialized_view_replaces_view(self, project, my_view): - super().test_materialized_view_replaces_view(project, my_view) - + @pytest.mark.skip( + "The current implementation does not support overwriting materialized views with tables." + ) def test_table_replaces_materialized_view(self, project, my_materialized_view): super().test_table_replaces_materialized_view(project, my_materialized_view) + @pytest.mark.skip( + "The current implementation does not support overwriting materialized views with views." + ) def test_view_replaces_materialized_view(self, project, my_materialized_view): super().test_view_replaces_materialized_view(project, my_materialized_view) + @pytest.mark.skip( + "The current implementation does not support overwriting tables with materialized views." + ) + def test_materialized_view_replaces_table(self, project, my_table): + super().test_materialized_view_replaces_table(project, my_table) + + @pytest.mark.skip( + "The current implementation does not support overwriting views with materialized views." + ) + def test_materialized_view_replaces_view(self, project, my_view): + super().test_materialized_view_replaces_view(project, my_view) + class PostgresMaterializedViewChanges(MaterializedViewChanges): @pytest.fixture(scope="class", autouse=True) diff --git a/tests/unit/test_relation.py b/tests/unit/test_relation.py deleted file mode 100644 index 3e0ebf7fac8..00000000000 --- a/tests/unit/test_relation.py +++ /dev/null @@ -1,19 +0,0 @@ -from dataclasses import replace - -import pytest - -from dbt.adapters.base import BaseRelation -from dbt.contracts.relation import RelationType - - -@pytest.mark.parametrize( - "relation_type,result", - [ - (RelationType.View, True), - (RelationType.Table, False), - ], -) -def test_can_be_renamed(relation_type, result): - my_relation = BaseRelation.create(type=relation_type) - my_relation = replace(my_relation, relations_that_can_be_renamed=[RelationType.View]) - assert my_relation.can_be_renamed is result From cefc7d4f0a94a6b7447a62e38e9b2c1f6150d9f8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 21 Aug 2023 13:19:37 -0400 Subject: [PATCH 10/11] add logging back to get_create_materialized_view_as_sql --- .../macros/adapters/relations/materialized_view/create.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/core/dbt/include/global_project/macros/adapters/relations/materialized_view/create.sql b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/create.sql index a0f2614c9ca..8b479964e6f 100644 --- a/core/dbt/include/global_project/macros/adapters/relations/materialized_view/create.sql +++ b/core/dbt/include/global_project/macros/adapters/relations/materialized_view/create.sql @@ -1,4 +1,5 @@ {% macro get_create_materialized_view_as_sql(relation, sql) -%} + {{- log('Applying CREATE to: ' ~ relation) -}} {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}} {%- endmacro %} From 49d02372ea639cab45661efc3c5deb1b1b58dee3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 21 Aug 2023 13:48:09 -0400 Subject: [PATCH 11/11] changie --- .changes/unreleased/Under the Hood-20230821-134801.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20230821-134801.yaml diff --git a/.changes/unreleased/Under the Hood-20230821-134801.yaml b/.changes/unreleased/Under the Hood-20230821-134801.yaml new file mode 100644 index 00000000000..eb2c09a2486 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230821-134801.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: 'Re-organize jinja macros: relation-specific in /macros/adapters/relations/, + relation agnostic in /macros/relations' +time: 2023-08-21T13:48:01.474731-04:00 +custom: + Author: mikealfare + Issue: "8449"