From c6ae62136ec272d9933b3a709ddf09a707eb459a Mon Sep 17 00:00:00 2001 From: James Zhao Date: Wed, 22 Dec 2021 10:19:48 +0800 Subject: [PATCH 01/16] add normalization-clickhouse docker build step --- airbyte-integrations/bases/base-normalization/build.gradle | 6 ++++++ .../bases/base-normalization/docker-compose.yaml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index ccdda2ce39dd4..eff597c11a3e1 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -69,10 +69,15 @@ task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) { configure buildAirbyteDocker('oracle') dependsOn assemble } +task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('clickhouse') + dependsOn assemble +} airbyteDocker.dependsOn(airbyteDockerMSSql) airbyteDocker.dependsOn(airbyteDockerMySql) airbyteDocker.dependsOn(airbyteDockerOracle) +airbyteDocker.dependsOn(airbyteDockerClickhouse) task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) { module = "pytest" @@ -86,6 +91,7 @@ task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' } integrationTest.dependsOn("customIntegrationTestPython") diff --git a/airbyte-integrations/bases/base-normalization/docker-compose.yaml b/airbyte-integrations/bases/base-normalization/docker-compose.yaml index ced1d036d206f..8dd94275765be 100644 --- a/airbyte-integrations/bases/base-normalization/docker-compose.yaml +++ b/airbyte-integrations/bases/base-normalization/docker-compose.yaml @@ -10,3 +10,5 @@ services: image: airbyte/normalization-mysql:${VERSION} normalization-oracle: image: airbyte/normalization-oracle:${VERSION} + normalization-clickhouse: + image: airbyte/normalization-clickhouse:${VERSION} From cc499c29fe83006e30d572f41e4dfc00ed89d5f1 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 22 Dec 2021 23:34:26 -0300 Subject: [PATCH 02/16] bump normalization version --- airbyte-integrations/bases/base-normalization/Dockerfile | 2 +- .../workers/normalization/NormalizationRunnerFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index a9efa7a9379f6..be915f01ab0d0 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.1.61 +LABEL io.airbyte.version=0.1.62 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index c2454395b6501..adcee9eb3b7a4 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.1.61"; + public static final String NORMALIZATION_VERSION = "0.1.62"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() From a2517a341d31012d579ccd584019bf3b06c3c949 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 23 Dec 2021 14:41:33 -0300 Subject: [PATCH 03/16] small changes gradle --- airbyte-integrations/bases/base-normalization/.dockerignore | 1 + build.gradle | 1 + 2 files changed, 2 insertions(+) diff --git a/airbyte-integrations/bases/base-normalization/.dockerignore b/airbyte-integrations/bases/base-normalization/.dockerignore index 2e284d6c45646..f1966e3f45ab5 100644 --- a/airbyte-integrations/bases/base-normalization/.dockerignore +++ b/airbyte-integrations/bases/base-normalization/.dockerignore @@ -8,3 +8,4 @@ !dbt-project-template-mssql !dbt-project-template-mysql !dbt-project-template-oracle +!dbt-project-template-clickhouse \ No newline at end of file diff --git a/build.gradle b/build.gradle index 33d14f4f50e56..c01e5aa9d9350 100644 --- a/build.gradle +++ b/build.gradle @@ -82,6 +82,7 @@ def createSpotlessTarget = { pattern -> 'dbt-project-template-mssql', 'dbt-project-template-mysql', 'dbt-project-template-oracle', + 'dbt-project-template-clickhouse', 'dbt_test_config', 'normalization_test_output', 'tools', From a57495c8d14b961fe0ca3802d60252aa9a480f05 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Mon, 27 Dec 2021 14:41:33 -0300 Subject: [PATCH 04/16] fix settings gradle --- settings.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/settings.gradle b/settings.gradle index 56da8a917448b..b50a3f1933dea 100644 --- a/settings.gradle +++ b/settings.gradle @@ -97,6 +97,7 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD" include ':airbyte-integrations:connectors:destination-snowflake' include ':airbyte-integrations:connectors:destination-oracle' include ':airbyte-integrations:connectors:destination-mssql' + include ':airbyte-integrations:connectors:destination-clickhouse' //Needed by destination-bigquery include ':airbyte-integrations:connectors:destination-s3' From c56ef5409e8dd44c7169b76240c046b7e54eb839 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Mon, 27 Dec 2021 14:42:46 -0300 Subject: [PATCH 05/16] fix eof file --- airbyte-integrations/bases/base-normalization/.dockerignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-normalization/.dockerignore b/airbyte-integrations/bases/base-normalization/.dockerignore index f1966e3f45ab5..e6fbfb3101a91 100644 --- a/airbyte-integrations/bases/base-normalization/.dockerignore +++ b/airbyte-integrations/bases/base-normalization/.dockerignore @@ -8,4 +8,4 @@ !dbt-project-template-mssql !dbt-project-template-mysql !dbt-project-template-oracle -!dbt-project-template-clickhouse \ No newline at end of file +!dbt-project-template-clickhouse From f8ccfd6df332a5f494fabaaaaa4ba8a31bee9e6f Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Tue, 28 Dec 2021 21:44:48 -0300 Subject: [PATCH 06/16] correct clickhouse normalization --- .../dbt_project.yml | 3 ++ .../dbt_project.yml | 1 + .../dbt_project.yml | 1 + .../dbt_project.yml | 1 + .../dbt-project-template/dbt_project.yml | 3 ++ .../test_simple_streams/dbt_project.yml | 3 ++ .../test_simple_streams/dbt_project.yml | 1 + .../test_nested_streams/dbt_project.yml | 3 ++ .../test_simple_streams/dbt_project.yml | 3 ++ .../test_simple_streams/first_dbt_project.yml | 3 ++ .../transform_catalog/stream_processor.py | 28 +++++++++++++++++-- 11 files changed, 47 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-clickhouse/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-clickhouse/dbt_project.yml index 37eca7fcd73f3..ae74081e6a4e9 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-clickhouse/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-clickhouse/dbt_project.yml @@ -62,3 +62,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "clickhouse" \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-mssql/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-mssql/dbt_project.yml index 8c7494fdc58f5..111260bb1df7d 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-mssql/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-mssql/dbt_project.yml @@ -59,3 +59,4 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] + destination: "mssql" diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml index b03cee8fe9305..dcf0bcf8e1885 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml @@ -61,3 +61,4 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] + destination: "myssql" diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-oracle/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-oracle/dbt_project.yml index 0ded2a42d60e9..4ed6a85b82a44 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-oracle/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-oracle/dbt_project.yml @@ -59,3 +59,4 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] + destination: "oracle" diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template/dbt_project.yml index 9ad8158759001..7ad0c2ea3ed5f 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml index 37eca7fcd73f3..ae74081e6a4e9 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml @@ -62,3 +62,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "clickhouse" \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml index 0ded2a42d60e9..4ed6a85b82a44 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml @@ -59,3 +59,4 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] + destination: "oracle" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml index 9ad8158759001..7ad0c2ea3ed5f 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml index 522b1e595e8ac..7568c2e4aea40 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml index 9ad8158759001..7ad0c2ea3ed5f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index e41b683b6b1bc..84f75ff3e8b41 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -699,6 +699,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup {{ sql_table_comment }} ), {{ '{% endif %}' }} +{{ '{%- if var("destination") == "clickhouse" %}' }} input_data_with_active_row_num as ( select *, row_number() over ( @@ -710,6 +711,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup ) as _airbyte_active_row_num from input_data ), +{{ '{%- endif %}' }} scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select @@ -725,7 +727,9 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup {{ field }}, {%- endfor %} {{ cursor_field }} as {{ airbyte_start_at }}, - {{ lag_begin }}({{ cursor_field }}) over ( + {{ '{%- if var("destination") == "clickhouse" %}' }} + case when _airbyte_active_row_num = 1{{ cdc_active_row }} then 1 else 0 end as {{ active_row }}, + {{ lag_begin }}({{ cursor_field }}) over ( partition by {{ primary_key_partition | join(", ") }} order by {{ cursor_field }} {{ order_null }}, @@ -733,11 +737,26 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup {{ col_emitted_at }} desc{{ cdc_updated_at_order }} {{ lag_end }} ) as {{ airbyte_end_at }}, - case when _airbyte_active_row_num = 1{{ cdc_active_row }} then 1 else 0 end as {{ active_row }}, + {{ '{%- else %}' }} + lag({{ cursor_field }}) over ( + partition by {{ primary_key_partition | join(", ") }} + order by + {{ cursor_field }} {{ order_null }}, + {{ cursor_field }} desc, + {{ col_emitted_at }} desc{{ cdc_updated_at_order }} + ) as {{ airbyte_end_at }}, + case when row_number() over ( + partition by {{ primary_key_partition | join(", ") }} + order by + {{ cursor_field }} {{ order_null }}, + {{ cursor_field }} desc, + {{ col_emitted_at }} desc{{ cdc_updated_at_order }} + ) = 1{{ cdc_active_row }} then 1 else 0 end as {{ active_row }}, + {{ '{%- endif %}' }} {{ col_ab_id }}, {{ col_emitted_at }}, {{ hash_id }} - from input_data_with_active_row_num + from {{ input_data_table }} ), dedup_data as ( select @@ -784,11 +803,13 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup lag_begin = "lag" lag_end = "" + input_data_table = "input_data" if self.destination_type == DestinationType.CLICKHOUSE: # ClickHouse doesn't support lag() yet, this is a workaround solution # Ref: https://clickhouse.com/docs/en/sql-reference/window-functions/ lag_begin = "anyOrNull" lag_end = "ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING" + input_data_table = "input_data_with_active_row_num" enable_left_join_null = "" cast_begin = "cast(" @@ -859,6 +880,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup lag_begin=lag_begin, lag_end=lag_end, enable_left_join_null=enable_left_join_null, + input_data_table=input_data_table, ) return sql From 82b77d3e33e33254214c9bc2c2851bf4234e87da Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Mon, 3 Jan 2022 18:35:49 +0100 Subject: [PATCH 07/16] add test case --- .../resources/test_simple_streams/data_input/catalog.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json index c8efc48f5b3e3..9a13299d088e4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json @@ -136,7 +136,7 @@ "default_cursor_field": [] }, "sync_mode": "incremental", - "cursor_field": [], + "cursor_field": ["_ab_cdc_lsn"], "destination_sync_mode": "append_dedup", "primary_key": [["id"]] }, From df3faf645b5b2b5928aa55c6c992d6c15efa27d2 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Mon, 20 Dec 2021 14:54:10 +0100 Subject: [PATCH 08/16] Cast cursor to string --- .../normalization/transform_catalog/stream_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index e41b683b6b1bc..e99febc66977e 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -744,7 +744,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by {{ unique_key }}, {{ airbyte_start_at }}, {{ col_emitted_at }}{{ cdc_cols }} + partition by {{ unique_key }}, cast({{ airbyte_start_at }} as {{ dbt_utils.type_string() }}), {{ col_emitted_at }}{{ cdc_cols }} order by {{ active_row }} desc, {{ col_ab_id }} ) as {{ airbyte_row_num }}, {{ '{{' }} dbt_utils.surrogate_key([ From a76ba27755a448a68db295520345a71886b0e0a0 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Tue, 4 Jan 2022 13:12:57 +0100 Subject: [PATCH 09/16] Refactor jinja template for scd --- .../transform_catalog/stream_processor.py | 273 +++++++++--------- 1 file changed, 142 insertions(+), 131 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 84f75ff3e8b41..1a8993ddf8cc3 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -649,7 +649,144 @@ def safe_cast_to_string(definition: Dict, column_name: str, destination_type: De return col def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tuple[str, str]]) -> str: - scd_sql_template = """ + order_null = "is null asc" + if self.destination_type.value == DestinationType.ORACLE.value: + order_null = "asc nulls last" + if self.destination_type.value == DestinationType.MSSQL.value: + # SQL Server treats NULL values as the lowest values, then sorted in ascending order, NULLs come first. + order_null = "desc" + + lag_begin = "lag" + lag_end = "" + input_data_table = "input_data" + if self.destination_type == DestinationType.CLICKHOUSE: + # ClickHouse doesn't support lag() yet, this is a workaround solution + # Ref: https://clickhouse.com/docs/en/sql-reference/window-functions/ + lag_begin = "anyOrNull" + lag_end = "ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING" + input_data_table = "input_data_with_active_row_num" + + enable_left_join_null = "" + cast_begin = "cast(" + cast_as = " as " + cast_end = ")" + if self.destination_type == DestinationType.CLICKHOUSE: + enable_left_join_null = "--" + cast_begin = "accurateCastOrNull(" + cast_as = ", '" + cast_end = "')" + + # TODO move all cdc columns out of scd models + cdc_active_row_pattern = "" + cdc_updated_order_pattern = "" + cdc_cols = "" + quoted_cdc_cols = "" + if "_ab_cdc_deleted_at" in column_names.keys(): + col_cdc_deleted_at = self.name_transformer.normalize_column_name("_ab_cdc_deleted_at") + col_cdc_updated_at = self.name_transformer.normalize_column_name("_ab_cdc_updated_at") + quoted_col_cdc_deleted_at = self.name_transformer.normalize_column_name("_ab_cdc_deleted_at", in_jinja=True) + quoted_col_cdc_updated_at = self.name_transformer.normalize_column_name("_ab_cdc_updated_at", in_jinja=True) + cdc_active_row_pattern = f" and {col_cdc_deleted_at} is null" + cdc_updated_order_pattern = f", {col_cdc_updated_at} desc" + cdc_cols = ( + f", {cast_begin}{col_cdc_deleted_at}{cast_as}" + + "{{ dbt_utils.type_string() }}" + + f"{cast_end}" + + f", {cast_begin}{col_cdc_updated_at}{cast_as}" + + "{{ dbt_utils.type_string() }}" + + f"{cast_end}" + ) + quoted_cdc_cols = f", {quoted_col_cdc_deleted_at}, {quoted_col_cdc_updated_at}" + + if "_ab_cdc_log_pos" in column_names.keys(): + col_cdc_log_pos = self.name_transformer.normalize_column_name("_ab_cdc_log_pos") + quoted_col_cdc_log_pos = self.name_transformer.normalize_column_name("_ab_cdc_log_pos", in_jinja=True) + cdc_updated_order_pattern += f", {col_cdc_log_pos} desc" + cdc_cols += f", {cast_begin}{col_cdc_log_pos}{cast_as}" + "{{ dbt_utils.type_string() }}" + f"{cast_end}" + quoted_cdc_cols += f", {quoted_col_cdc_log_pos}" + + jinja_variables = { + "active_row": self.name_transformer.normalize_column_name("_airbyte_active_row"), + "airbyte_end_at": self.name_transformer.normalize_column_name("_airbyte_end_at"), + "airbyte_row_num": self.name_transformer.normalize_column_name("_airbyte_row_num"), + "airbyte_start_at": self.name_transformer.normalize_column_name("_airbyte_start_at"), + "airbyte_unique_key_scd": self.name_transformer.normalize_column_name(f"{self.airbyte_unique_key}_scd"), + "cdc_active_row": cdc_active_row_pattern, + "cdc_cols": cdc_cols, + "cdc_updated_at_order": cdc_updated_order_pattern, + "col_ab_id": self.get_ab_id(), + "col_emitted_at": self.get_emitted_at(), + "col_normalized_at": self.get_normalized_at(), + "cursor_field": self.get_cursor_field(column_names), + "enable_left_join_null": enable_left_join_null, + "fields": self.list_fields(column_names), + "from_table": from_table, + "hash_id": self.hash_id(), + "input_data_table": input_data_table, + "lag_begin": lag_begin, + "lag_end": lag_end, + "order_null": order_null, + "parent_hash_id": self.parent_hash_id(), + "primary_key_partition": self.get_primary_key_partition(column_names), + "primary_keys": self.list_primary_keys(column_names), + "quoted_airbyte_row_num": self.name_transformer.normalize_column_name("_airbyte_row_num", in_jinja=True), + "quoted_airbyte_start_at": self.name_transformer.normalize_column_name("_airbyte_start_at", in_jinja=True), + "quoted_cdc_cols": quoted_cdc_cols, + "quoted_col_emitted_at": self.get_emitted_at(in_jinja=True), + "quoted_unique_key": self.get_unique_key(in_jinja=True), + "sql_table_comment": self.sql_table_comment(include_from_table=True), + "unique_key": self.get_unique_key(), + } + if self.destination_type == DestinationType.CLICKHOUSE: + clickhouse_active_row_sql = Template( + """ +input_data_with_active_row_num as ( + select *, + row_number() over ( + partition by {{ primary_key_partition | join(", ") }} + order by + {{ cursor_field }} {{ order_null }}, + {{ cursor_field }} desc, + {{ col_emitted_at }} desc{{ cdc_updated_at_order }} + ) as _airbyte_active_row_num + from input_data +),""" + ).render(jinja_variables) + jinja_variables["clickhouse_active_row_sql"] = clickhouse_active_row_sql + scd_columns_sql = Template( + """ + case when _airbyte_active_row_num = 1{{ cdc_active_row }} then 1 else 0 end as {{ active_row }}, + {{ lag_begin }}({{ cursor_field }}) over ( + partition by {{ primary_key_partition | join(", ") }} + order by + {{ cursor_field }} {{ order_null }}, + {{ cursor_field }} desc, + {{ col_emitted_at }} desc{{ cdc_updated_at_order }} + {{ lag_end }} + ) as {{ airbyte_end_at }}""" + ).render(jinja_variables) + jinja_variables["scd_columns_sql"] = scd_columns_sql + else: + scd_columns_sql = Template( + """ + lag({{ cursor_field }}) over ( + partition by {{ primary_key_partition | join(", ") }} + order by + {{ cursor_field }} {{ order_null }}, + {{ cursor_field }} desc, + {{ col_emitted_at }} desc{{ cdc_updated_at_order }} + ) as {{ airbyte_end_at }}, + case when row_number() over ( + partition by {{ primary_key_partition | join(", ") }} + order by + {{ cursor_field }} {{ order_null }}, + {{ cursor_field }} desc, + {{ col_emitted_at }} desc{{ cdc_updated_at_order }} + ) = 1{{ cdc_active_row }} then 1 else 0 end as {{ active_row }}""" + ).render(jinja_variables) + jinja_variables["scd_columns_sql"] = scd_columns_sql + sql = Template( + """ -- depends_on: {{ from_table }} with {{ '{% if is_incremental() %}' }} @@ -699,19 +836,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup {{ sql_table_comment }} ), {{ '{% endif %}' }} -{{ '{%- if var("destination") == "clickhouse" %}' }} -input_data_with_active_row_num as ( - select *, - row_number() over ( - partition by {{ primary_key_partition | join(", ") }} - order by - {{ cursor_field }} {{ order_null }}, - {{ cursor_field }} desc, - {{ col_emitted_at }} desc{{ cdc_updated_at_order }} - ) as _airbyte_active_row_num - from input_data -), -{{ '{%- endif %}' }} +{{ clickhouse_active_row_sql }} scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select @@ -727,32 +852,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup {{ field }}, {%- endfor %} {{ cursor_field }} as {{ airbyte_start_at }}, - {{ '{%- if var("destination") == "clickhouse" %}' }} - case when _airbyte_active_row_num = 1{{ cdc_active_row }} then 1 else 0 end as {{ active_row }}, - {{ lag_begin }}({{ cursor_field }}) over ( - partition by {{ primary_key_partition | join(", ") }} - order by - {{ cursor_field }} {{ order_null }}, - {{ cursor_field }} desc, - {{ col_emitted_at }} desc{{ cdc_updated_at_order }} - {{ lag_end }} - ) as {{ airbyte_end_at }}, - {{ '{%- else %}' }} - lag({{ cursor_field }}) over ( - partition by {{ primary_key_partition | join(", ") }} - order by - {{ cursor_field }} {{ order_null }}, - {{ cursor_field }} desc, - {{ col_emitted_at }} desc{{ cdc_updated_at_order }} - ) as {{ airbyte_end_at }}, - case when row_number() over ( - partition by {{ primary_key_partition | join(", ") }} - order by - {{ cursor_field }} {{ order_null }}, - {{ cursor_field }} desc, - {{ col_emitted_at }} desc{{ cdc_updated_at_order }} - ) = 1{{ cdc_active_row }} then 1 else 0 end as {{ active_row }}, - {{ '{%- endif %}' }} + {{ scd_columns_sql }}, {{ col_ab_id }}, {{ col_emitted_at }}, {{ hash_id }} @@ -791,97 +891,8 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup {{ '{{ current_timestamp() }}' }} as {{ col_normalized_at }}, {{ hash_id }} from dedup_data where {{ airbyte_row_num }} = 1 - """ - template = Template(scd_sql_template) - - order_null = "is null asc" - if self.destination_type.value == DestinationType.ORACLE.value: - order_null = "asc nulls last" - if self.destination_type.value == DestinationType.MSSQL.value: - # SQL Server treats NULL values as the lowest values, then sorted in ascending order, NULLs come first. - order_null = "desc" - - lag_begin = "lag" - lag_end = "" - input_data_table = "input_data" - if self.destination_type == DestinationType.CLICKHOUSE: - # ClickHouse doesn't support lag() yet, this is a workaround solution - # Ref: https://clickhouse.com/docs/en/sql-reference/window-functions/ - lag_begin = "anyOrNull" - lag_end = "ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING" - input_data_table = "input_data_with_active_row_num" - - enable_left_join_null = "" - cast_begin = "cast(" - cast_as = " as " - cast_end = ")" - if self.destination_type == DestinationType.CLICKHOUSE: - enable_left_join_null = "--" - cast_begin = "accurateCastOrNull(" - cast_as = ", '" - cast_end = "')" - - # TODO move all cdc columns out of scd models - cdc_active_row_pattern = "" - cdc_updated_order_pattern = "" - cdc_cols = "" - quoted_cdc_cols = "" - if "_ab_cdc_deleted_at" in column_names.keys(): - col_cdc_deleted_at = self.name_transformer.normalize_column_name("_ab_cdc_deleted_at") - col_cdc_updated_at = self.name_transformer.normalize_column_name("_ab_cdc_updated_at") - quoted_col_cdc_deleted_at = self.name_transformer.normalize_column_name("_ab_cdc_deleted_at", in_jinja=True) - quoted_col_cdc_updated_at = self.name_transformer.normalize_column_name("_ab_cdc_updated_at", in_jinja=True) - cdc_active_row_pattern = f" and {col_cdc_deleted_at} is null" - cdc_updated_order_pattern = f", {col_cdc_updated_at} desc" - cdc_cols = ( - f", {cast_begin}{col_cdc_deleted_at}{cast_as}" - + "{{ dbt_utils.type_string() }}" - + f"{cast_end}" - + f", {cast_begin}{col_cdc_updated_at}{cast_as}" - + "{{ dbt_utils.type_string() }}" - + f"{cast_end}" - ) - quoted_cdc_cols = f", {quoted_col_cdc_deleted_at}, {quoted_col_cdc_updated_at}" - - if "_ab_cdc_log_pos" in column_names.keys(): - col_cdc_log_pos = self.name_transformer.normalize_column_name("_ab_cdc_log_pos") - quoted_col_cdc_log_pos = self.name_transformer.normalize_column_name("_ab_cdc_log_pos", in_jinja=True) - cdc_updated_order_pattern += f", {col_cdc_log_pos} desc" - cdc_cols += f", {cast_begin}{col_cdc_log_pos}{cast_as}" + "{{ dbt_utils.type_string() }}" + f"{cast_end}" - quoted_cdc_cols += f", {quoted_col_cdc_log_pos}" - - sql = template.render( - order_null=order_null, - airbyte_start_at=self.name_transformer.normalize_column_name("_airbyte_start_at"), - quoted_airbyte_start_at=self.name_transformer.normalize_column_name("_airbyte_start_at", in_jinja=True), - airbyte_end_at=self.name_transformer.normalize_column_name("_airbyte_end_at"), - active_row=self.name_transformer.normalize_column_name("_airbyte_active_row"), - airbyte_row_num=self.name_transformer.normalize_column_name("_airbyte_row_num"), - quoted_airbyte_row_num=self.name_transformer.normalize_column_name("_airbyte_row_num", in_jinja=True), - airbyte_unique_key_scd=self.name_transformer.normalize_column_name(f"{self.airbyte_unique_key}_scd"), - unique_key=self.get_unique_key(), - quoted_unique_key=self.get_unique_key(in_jinja=True), - col_ab_id=self.get_ab_id(), - col_emitted_at=self.get_emitted_at(), - quoted_col_emitted_at=self.get_emitted_at(in_jinja=True), - col_normalized_at=self.get_normalized_at(), - parent_hash_id=self.parent_hash_id(), - fields=self.list_fields(column_names), - cursor_field=self.get_cursor_field(column_names), - primary_keys=self.list_primary_keys(column_names), - primary_key_partition=self.get_primary_key_partition(column_names), - hash_id=self.hash_id(), - from_table=from_table, - sql_table_comment=self.sql_table_comment(include_from_table=True), - cdc_active_row=cdc_active_row_pattern, - cdc_updated_at_order=cdc_updated_order_pattern, - cdc_cols=cdc_cols, - quoted_cdc_cols=quoted_cdc_cols, - lag_begin=lag_begin, - lag_end=lag_end, - enable_left_join_null=enable_left_join_null, - input_data_table=input_data_table, - ) +""" + ).render(jinja_variables) return sql def get_cursor_field(self, column_names: Dict[str, Tuple[str, str]], in_jinja: bool = False) -> str: From 5e48f8ad8232e0a06b9b4ba5cf4d962f2a98181c Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Tue, 4 Jan 2022 14:28:03 +0100 Subject: [PATCH 10/16] Fix normalization scd in oracle --- .../transform_catalog/stream_processor.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 956c772d98aa8..e38f5260f969c 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -651,7 +651,7 @@ def safe_cast_to_string(definition: Dict, column_name: str, destination_type: De def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tuple[str, str]]) -> str: order_null = "is null asc" if self.destination_type.value == DestinationType.ORACLE.value: - order_null = "asc nulls last" + order_null = "desc nulls last" if self.destination_type.value == DestinationType.MSSQL.value: # SQL Server treats NULL values as the lowest values, then sorted in ascending order, NULLs come first. order_null = "desc" @@ -705,11 +705,27 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup cdc_cols += f", {cast_begin}{col_cdc_log_pos}{cast_as}" + "{{ dbt_utils.type_string() }}" + f"{cast_end}" quoted_cdc_cols += f", {quoted_col_cdc_log_pos}" + if ( + self.destination_type == DestinationType.BIGQUERY + and len(self.cursor_field) == 1 + and is_number(self.properties[self.cursor_field[0]]["type"]) + ): + # partition by float columns is not allowed in BigQuery, cast it to string + airbyte_start_at_string = ( + cast_begin + + self.name_transformer.normalize_column_name("_airbyte_start_at") + + cast_as + + "{{ dbt_utils.type_string() }}" + + cast_end + ) + else: + airbyte_start_at_string = self.name_transformer.normalize_column_name("_airbyte_start_at") jinja_variables = { "active_row": self.name_transformer.normalize_column_name("_airbyte_active_row"), "airbyte_end_at": self.name_transformer.normalize_column_name("_airbyte_end_at"), "airbyte_row_num": self.name_transformer.normalize_column_name("_airbyte_row_num"), "airbyte_start_at": self.name_transformer.normalize_column_name("_airbyte_start_at"), + "airbyte_start_at_string": airbyte_start_at_string, "airbyte_unique_key_scd": self.name_transformer.normalize_column_name(f"{self.airbyte_unique_key}_scd"), "cdc_active_row": cdc_active_row_pattern, "cdc_cols": cdc_cols, @@ -863,7 +879,10 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by {{ unique_key }}, cast({{ airbyte_start_at }} as {{ dbt_utils.type_string() }}), {{ col_emitted_at }}{{ cdc_cols }} + partition by + {{ unique_key }}, + {{ airbyte_start_at_string }}, + {{ col_emitted_at }}{{ cdc_cols }} order by {{ active_row }} desc, {{ col_ab_id }} ) as {{ airbyte_row_num }}, {{ '{{' }} dbt_utils.surrogate_key([ From fff86cc7341c2165b724d5bad567dc5794eabd65 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Tue, 4 Jan 2022 15:10:29 +0100 Subject: [PATCH 11/16] Regen sql --- .../test_nested_streams/dbt_project.yml | 3 +++ ...ex_columns_resulting_into_long_names_scd.sql | 5 ++++- ...ex_columns_resulting_into_long_names_scd.sql | 5 ++++- .../test_simple_streams/dbt_project.yml | 3 +++ .../test_simple_streams/first_dbt_project.yml | 3 +++ .../dedup_exchange_rate_scd.sql | 5 ++++- .../dedup_exchange_rate_scd.sql | 5 ++++- .../dedup_exchange_rate_scd.sql | 5 ++++- .../mssql/test_nested_streams/dbt_project.yml | 1 + ...tream_with_co__lting_into_long_names_scd.sql | 5 ++++- ...tream_with_co__lting_into_long_names_scd.sql | 5 ++++- .../mssql/test_simple_streams/dbt_project.yml | 1 + .../dedup_exchange_rate_scd.sql | 5 ++++- .../dedup_exchange_rate_scd.sql | 5 ++++- .../mysql/test_nested_streams/dbt_project.yml | 1 + ...ed_stream_with_co_1g_into_long_names_scd.sql | 5 ++++- ...ed_stream_with_co_1g_into_long_names_scd.sql | 5 ++++- ...ed_stream_with_co_1g_into_long_names_scd.sql | 5 ++++- .../mysql/test_simple_streams/dbt_project.yml | 1 + .../dedup_exchange_rate_scd.sql | 5 ++++- .../dedup_exchange_rate_scd.sql | 5 ++++- .../dedup_exchange_rate_scd.sql | 5 ++++- .../dedup_exchange_rate_scd.sql | 9 ++++++--- .../dedup_exchange_rate_scd.sql | 9 ++++++--- .../dedup_exchange_rate_scd.sql | 9 ++++++--- ...stream_with_c__lting_into_long_names_scd.sql | 5 ++++- .../some_stream_that_was_empty_scd.sql | 5 ++++- ...stream_with_c__lting_into_long_names_scd.sql | 5 ++++- .../some_stream_that_was_empty_scd.sql | 5 ++++- .../dedup_cdc_excluded_scd.sql | 17 ++++++++++------- .../dedup_exchange_rate_scd.sql | 5 ++++- .../test_normalization/pos_dedup_cdcx_scd.sql | 5 ++++- .../renamed_dedup_cdc_excluded_scd.sql | 5 ++++- .../dedup_cdc_excluded_scd.sql | 17 ++++++++++------- .../dedup_exchange_rate_scd.sql | 5 ++++- .../test_normalization/pos_dedup_cdcx_scd.sql | 5 ++++- .../renamed_dedup_cdc_excluded_scd.sql | 5 ++++- .../dedup_exchange_rate_scd.sql | 5 ++++- .../renamed_dedup_cdc_excluded_scd.sql | 5 ++++- .../test_nested_streams/dbt_project.yml | 3 +++ ...ex_columns_resulting_into_long_names_scd.sql | 5 ++++- ...ex_columns_resulting_into_long_names_scd.sql | 5 ++++- .../test_simple_streams/dbt_project.yml | 3 +++ .../test_simple_streams/first_dbt_project.yml | 3 +++ .../dedup_exchange_rate_scd.sql | 5 ++++- .../dedup_exchange_rate_scd.sql | 5 ++++- .../dedup_exchange_rate_scd.sql | 5 ++++- .../test_nested_streams/dbt_project.yml | 3 +++ ...EX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 5 ++++- ...EX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 5 ++++- .../test_simple_streams/dbt_project.yml | 3 +++ .../DEDUP_EXCHANGE_RATE_SCD.sql | 5 ++++- .../DEDUP_EXCHANGE_RATE_SCD.sql | 5 ++++- 53 files changed, 210 insertions(+), 59 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml index 9ad8158759001..7ad0c2ea3ed5f 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index c98dcfdb9a7ed..60a896682fffc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -54,7 +54,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 188d65104df21..1dd492613c5a7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -87,7 +87,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml index 522b1e595e8ac..7568c2e4aea40 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml index 9ad8158759001..7ad0c2ea3ed5f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 28f810d9152da..da4e350a2da76 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -67,7 +67,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index df5b57acadcc1..d31c177ae671a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -96,7 +96,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index bd4c0b6271b45..1284489491f70 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -96,7 +96,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml index 8c7494fdc58f5..111260bb1df7d 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml @@ -59,3 +59,4 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] + destination: "mssql" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql index ca7cb37338611..904d4721cbcd3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -67,7 +67,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, convert(varchar(32), HashBytes(''md5'', coalesce(cast( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql index f312cf3afbfd4..32f5b97aaff50 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -85,7 +85,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml index 8c7494fdc58f5..111260bb1df7d 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml @@ -59,3 +59,4 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] + destination: "mssql" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index a572299fff2d4..198c37bf1797f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -76,7 +76,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, convert(varchar(32), HashBytes(''md5'', coalesce(cast( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 3bd5d5d499ed7..48fa805a9ce7f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -94,7 +94,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml index b03cee8fe9305..dcf0bcf8e1885 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml @@ -61,3 +61,4 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] + destination: "myssql" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index 1e688d24d74f1..51184a5a90a23 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -45,7 +45,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index 78b7de1f601f7..9f7abb50d9d30 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -85,7 +85,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index 1e688d24d74f1..51184a5a90a23 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -45,7 +45,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml index b03cee8fe9305..dcf0bcf8e1885 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml @@ -61,3 +61,4 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] + destination: "myssql" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 64ded010aa63b..30eb801075502 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -50,7 +50,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index abfaa2002b55f..ed12ca056e793 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -94,7 +94,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 64ded010aa63b..30eb801075502 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -50,7 +50,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index cbca1a34898ff..db458b497ca2f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -39,14 +39,14 @@ scd_data as ( lag("DATE") over ( partition by id, currency, cast(nzd as varchar2(4000)) order by - "DATE" asc nulls last, + "DATE" desc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) as "_AIRBYTE_END_AT", case when row_number() over ( partition by id, currency, cast(nzd as varchar2(4000)) order by - "DATE" asc nulls last, + "DATE" desc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) = 1 then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", @@ -60,7 +60,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + partition by + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_START_AT", + "_AIRBYTE_EMITTED_AT" order by "_AIRBYTE_ACTIVE_ROW" desc, "_AIRBYTE_AB_ID" ) as "_AIRBYTE_ROW_NUM", ora_hash( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 09ada8d511715..bd2717a7a654d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -73,14 +73,14 @@ scd_data as ( lag({{ quote('DATE') }}) over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by - {{ quote('DATE') }} asc nulls last, + {{ quote('DATE') }} desc nulls last, {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc ) as {{ quote('_AIRBYTE_END_AT') }}, case when row_number() over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by - {{ quote('DATE') }} asc nulls last, + {{ quote('DATE') }} desc nulls last, {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc ) = 1 then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, @@ -94,7 +94,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }} + partition by + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + {{ quote('_AIRBYTE_START_AT') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }} order by {{ quote('_AIRBYTE_ACTIVE_ROW') }} desc, {{ quote('_AIRBYTE_AB_ID') }} ) as {{ quote('_AIRBYTE_ROW_NUM') }}, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index cbca1a34898ff..db458b497ca2f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -39,14 +39,14 @@ scd_data as ( lag("DATE") over ( partition by id, currency, cast(nzd as varchar2(4000)) order by - "DATE" asc nulls last, + "DATE" desc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) as "_AIRBYTE_END_AT", case when row_number() over ( partition by id, currency, cast(nzd as varchar2(4000)) order by - "DATE" asc nulls last, + "DATE" desc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) = 1 then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", @@ -60,7 +60,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + partition by + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_START_AT", + "_AIRBYTE_EMITTED_AT" order by "_AIRBYTE_ACTIVE_ROW" desc, "_AIRBYTE_AB_ID" ) as "_AIRBYTE_ROW_NUM", ora_hash( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql index 5ceb844df0ae3..14c58318d53e1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -49,7 +49,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(coalesce(cast(_airbyte_unique_key as diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql index c1b4813412748..8ad5609bf2134 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql @@ -48,7 +48,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(coalesce(cast(_airbyte_unique_key as diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql index 3f1ab268c7d40..ee15d789a7d4d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -86,7 +86,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql index d6592cc28f8fa..3194631fa65f9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql @@ -85,7 +85,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index 450c207deea6c..c0c8a4a752d0b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -26,19 +26,19 @@ scd_data as ( _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( + _ab_cdc_lsn as _airbyte_start_at, + lag(_ab_cdc_lsn) over ( partition by "id" order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_lsn is null asc, + _ab_cdc_lsn desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc ) as _airbyte_end_at, case when row_number() over ( partition by "id" order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_lsn is null asc, + _ab_cdc_lsn desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, @@ -51,7 +51,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar ), cast(_ab_cdc_updated_at as varchar diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index a44a3ee1ffd4f..a92502150e8b4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -62,7 +62,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(coalesce(cast(_airbyte_unique_key as diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql index a1466c6f433d8..a4541da8b27bd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -52,7 +52,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar ), cast(_ab_cdc_updated_at as varchar diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 993d6b15efad0..0f7b4b8de97ac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -47,7 +47,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(coalesce(cast(_airbyte_unique_key as diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index d63cd07a8a4dd..50f9ec0632708 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -63,19 +63,19 @@ scd_data as ( _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( + _ab_cdc_lsn as _airbyte_start_at, + lag(_ab_cdc_lsn) over ( partition by {{ adapter.quote('id') }} order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_lsn is null asc, + _ab_cdc_lsn desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc ) as _airbyte_end_at, case when row_number() over ( partition by {{ adapter.quote('id') }} order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_lsn is null asc, + _ab_cdc_lsn desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, @@ -88,7 +88,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 3a2d131cb654c..d1e9d599e25f3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -95,7 +95,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql index 61210186eb00c..213f78fd53691 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -89,7 +89,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 89a414852dbc8..5c9e9fa82cf35 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -84,7 +84,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 4fa8039ff1a85..8a71da096568d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -95,7 +95,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index f88aa6e23f3b5..efc5b2c1671b6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -88,7 +88,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml index 9ad8158759001..7ad0c2ea3ed5f 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index f56ebb0d393ad..0ed248505d323 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -49,7 +49,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index f63d93b3787bc..695f5595a996e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -86,7 +86,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml index 522b1e595e8ac..7568c2e4aea40 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml index 9ad8158759001..7ad0c2ea3ed5f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 6550ed24e0699..69cafeae2f223 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -54,7 +54,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 28eaf5da09dd3..fbbbd04bae94f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -95,7 +95,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 24c88aab4c0d3..3a93bd7be1962 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -95,7 +95,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml index 9ad8158759001..7ad0c2ea3ed5f 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql index 7bc1f2d4776b6..e360f41b2c90a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -48,7 +48,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + partition by + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_START_AT, + _AIRBYTE_EMITTED_AT order by _AIRBYTE_ACTIVE_ROW desc, _AIRBYTE_AB_ID ) as _AIRBYTE_ROW_NUM, md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql index 0333595a99b97..738a31e2d2969 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -86,7 +86,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + partition by + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_START_AT, + _AIRBYTE_EMITTED_AT order by _AIRBYTE_ACTIVE_ROW desc, _AIRBYTE_AB_ID ) as _AIRBYTE_ROW_NUM, {{ dbt_utils.surrogate_key([ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml index 9ad8158759001..7ad0c2ea3ed5f 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml @@ -61,3 +61,6 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] + +vars: + destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql index 804ca297a46fd..098f6d3e91026 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -61,7 +61,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + partition by + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_START_AT, + _AIRBYTE_EMITTED_AT order by _AIRBYTE_ACTIVE_ROW desc, _AIRBYTE_AB_ID ) as _AIRBYTE_ROW_NUM, md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql index ca5c91ab9c9be..241db9acb9984 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -95,7 +95,10 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + partition by + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_START_AT, + _AIRBYTE_EMITTED_AT order by _AIRBYTE_ACTIVE_ROW desc, _AIRBYTE_AB_ID ) as _AIRBYTE_ROW_NUM, {{ dbt_utils.surrogate_key([ From 1d9dfb6e1a2166986aa7f05af09b9c9b44357832 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Tue, 4 Jan 2022 17:31:28 +0100 Subject: [PATCH 12/16] Regen clickhouse --- .../dedup_cdc_excluded_scd.sql | 26 ++++--- .../dedup_exchange_rate_scd.sql | 14 ++-- .../renamed_dedup_cdc_excluded_scd.sql | 14 ++-- .../test_normalization/dedup_cdc_excluded.sql | 1 + .../dedup_exchange_rate.sql | 1 + .../renamed_dedup_cdc_excluded.sql | 1 + .../test_normalization/exchange_rate.sql | 4 + .../dedup_cdc_excluded_ab3.sql | 71 ----------------- .../dedup_exchange_rate_stg.sql} | 5 +- .../test_normalization/pos_dedup_cdcx_ab3.sql | 78 ------------------- .../renamed_dedup_cdc_excluded_ab3.sql | 45 ----------- .../dedup_cdc_excluded_ab1.sql | 1 + .../dedup_cdc_excluded_ab2.sql | 1 + .../dedup_exchange_rate_ab1.sql | 1 + .../dedup_exchange_rate_ab2.sql | 1 + .../test_normalization/exchange_rate_ab1.sql | 1 + .../test_normalization/exchange_rate_ab2.sql | 1 + .../test_normalization/exchange_rate_ab3.sql | 1 + .../test_normalization/pos_dedup_cdcx_ab1.sql | 1 + .../test_normalization/pos_dedup_cdcx_ab2.sql | 1 + .../renamed_dedup_cdc_excluded_ab1.sql | 1 + .../renamed_dedup_cdc_excluded_ab2.sql | 1 + .../dedup_cdc_excluded_scd.sql | 43 ++++++---- .../dedup_exchange_rate_scd.sql | 31 +++++--- .../renamed_dedup_cdc_excluded_scd.sql | 31 +++++--- .../test_normalization/dedup_cdc_excluded.sql | 1 + .../dedup_exchange_rate.sql | 1 + .../renamed_dedup_cdc_excluded.sql | 1 + .../test_normalization/exchange_rate.sql | 1 + .../dedup_cdc_excluded_ab3.sql | 20 ----- ...te_ab3.sql => dedup_exchange_rate_stg.sql} | 1 + .../test_normalization/pos_dedup_cdcx_ab3.sql | 21 ----- .../renamed_dedup_cdc_excluded_ab3.sql | 16 ---- .../test_normalization/exchange_rate.sql | 4 + .../dedup_cdc_excluded_ab3.sql | 71 ----------------- .../dedup_exchange_rate_stg.sql} | 5 +- .../test_normalization/pos_dedup_cdcx_ab3.sql | 78 ------------------- .../renamed_dedup_cdc_excluded_ab3.sql | 45 ----------- 38 files changed, 134 insertions(+), 507 deletions(-) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/{second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql => first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql} (91%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/{dedup_exchange_rate_ab3.sql => dedup_exchange_rate_stg.sql} (92%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/{first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql => second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql} (91%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index 749e5b38562b4..ca78582b180d9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -12,11 +12,12 @@ as ( +-- depends_on: ref('dedup_cdc_excluded_stg') with input_data as ( select * - from _airbyte_test_normalization.dedup_cdc_excluded_ab3 + from _airbyte_test_normalization.dedup_cdc_excluded_stg -- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded ), @@ -25,8 +26,8 @@ input_data_with_active_row_num as ( row_number() over ( partition by id order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_lsn is null asc, + _ab_cdc_lsn desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc ) as _airbyte_active_row_num from input_data @@ -44,16 +45,16 @@ scd_data as ( _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - anyOrNull(_airbyte_emitted_at) over ( + _ab_cdc_lsn as _airbyte_start_at, + case when _airbyte_active_row_num = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + anyOrNull(_ab_cdc_lsn) over ( partition by id order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_lsn is null asc, + _ab_cdc_lsn desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING ) as _airbyte_end_at, - case when _airbyte_active_row_num = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_cdc_excluded_hashid @@ -64,8 +65,11 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, accurateCastOrNull(_ab_cdc_deleted_at, 'String'), accurateCastOrNull(_ab_cdc_updated_at, 'String') - order by _airbyte_ab_id + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at, accurateCastOrNull(_ab_cdc_deleted_at, 'String'), accurateCastOrNull(_ab_cdc_updated_at, 'String') + order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, assumeNotNull(hex(MD5( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 49c1843204fee..93e86c0b32d3b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -12,11 +12,12 @@ as ( +-- depends_on: ref('dedup_exchange_rate_stg') with input_data as ( select * - from _airbyte_test_normalization.dedup_exchange_rate_ab3 + from _airbyte_test_normalization.dedup_exchange_rate_stg -- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate ), @@ -54,15 +55,15 @@ scd_data as ( NZD, USD, date as _airbyte_start_at, + case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, anyOrNull(date) over ( partition by id, currency, cast(NZD as String) order by date is null asc, date desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING ) as _airbyte_end_at, - case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid @@ -73,8 +74,11 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at + order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, assumeNotNull(hex(MD5( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 61f46aa4665c4..d96c53f97da05 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -12,11 +12,12 @@ as ( +-- depends_on: ref('renamed_dedup_cdc_excluded_stg') with input_data as ( select * - from _airbyte_test_normalization.renamed_dedup_cdc_excluded_ab3 + from _airbyte_test_normalization.renamed_dedup_cdc_excluded_stg -- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded ), @@ -41,15 +42,15 @@ scd_data as ( ))) as _airbyte_unique_key, id, _airbyte_emitted_at as _airbyte_start_at, + case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, anyOrNull(_airbyte_emitted_at) over ( partition by id order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING ) as _airbyte_end_at, - case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_renamed_dedup_cdc_excluded_hashid @@ -60,8 +61,11 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at + order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, assumeNotNull(hex(MD5( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index 4b2055de8600d..8aea31930d35c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -13,6 +13,7 @@ as ( -- Final base SQL model +-- depends_on: test_normalization.dedup_cdc_excluded_scd select _airbyte_unique_key, id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index d1dc1abdc7142..28204615e97cf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -13,6 +13,7 @@ as ( -- Final base SQL model +-- depends_on: test_normalization.dedup_exchange_rate_scd select _airbyte_unique_key, id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index 5295b9baa8dae..b16b5361120f0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -13,6 +13,7 @@ as ( -- Final base SQL model +-- depends_on: test_normalization.renamed_dedup_cdc_excluded_scd select _airbyte_unique_key, id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index 0eb15bc43e455..2ee3d293b8403 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -14,6 +14,7 @@ with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: test_normalization._airbyte_raw_exchange_rate select JSONExtractRaw(_airbyte_data, 'id') as id, JSONExtractRaw(_airbyte_data, 'currency') as currency, @@ -33,6 +34,7 @@ where 1 = 1 ), __dbt__cte__exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: __dbt__cte__exchange_rate_ab1 select accurateCastOrNull(id, ' BIGINT @@ -60,6 +62,7 @@ where 1 = 1 ), __dbt__cte__exchange_rate_ab3 as ( -- SQL model to build a hash column based on the values of this record +-- depends_on: __dbt__cte__exchange_rate_ab2 select assumeNotNull(hex(MD5( @@ -95,6 +98,7 @@ from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate where 1 = 1 )-- Final base SQL model +-- depends_on: __dbt__cte__exchange_rate_ab3 select id, currency, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index fe2bf632dbf20..0000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,71 +0,0 @@ - - - create view _airbyte_test_normalization.dedup_cdc_excluded_ab3__dbt_tmp - - as ( - -with __dbt__cte__dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - JSONExtractRaw(_airbyte_data, 'id') as id, - JSONExtractRaw(_airbyte_data, 'name') as name, - JSONExtractRaw(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - JSONExtractRaw(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - JSONExtractRaw(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - -), __dbt__cte__dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - accurateCastOrNull(id, ' - BIGINT -') as id, - nullif(accurateCastOrNull(trim(BOTH '"' from name), 'String'), 'null') as name, - accurateCastOrNull(_ab_cdc_lsn, ' - Float64 -') as _ab_cdc_lsn, - accurateCastOrNull(_ab_cdc_updated_at, ' - Float64 -') as _ab_cdc_updated_at, - accurateCastOrNull(_ab_cdc_deleted_at, ' - Float64 -') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from __dbt__cte__dedup_cdc_excluded_ab1 --- dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - assumeNotNull(hex(MD5( - - toString(id) || '~' || - - - toString(name) || '~' || - - - toString(_ab_cdc_lsn) || '~' || - - - toString(_ab_cdc_updated_at) || '~' || - - - toString(_ab_cdc_deleted_at) - - ))) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__dedup_cdc_excluded_ab2 tmp --- dedup_cdc_excluded -where 1 = 1 - - ) \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 28abd1a79a7f2..799af4ec78aba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,12 +1,13 @@ - create view _airbyte_test_normalization.dedup_exchange_rate_ab3__dbt_tmp + create view _airbyte_test_normalization.dedup_exchange_rate_stg__dbt_tmp as ( with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate select JSONExtractRaw(_airbyte_data, 'id') as id, JSONExtractRaw(_airbyte_data, 'currency') as currency, @@ -26,6 +27,7 @@ where 1 = 1 ), __dbt__cte__dedup_exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: __dbt__cte__dedup_exchange_rate_ab1 select accurateCastOrNull(id, ' BIGINT @@ -51,6 +53,7 @@ from __dbt__cte__dedup_exchange_rate_ab1 where 1 = 1 )-- SQL model to build a hash column based on the values of this record +-- depends_on: __dbt__cte__dedup_exchange_rate_ab2 select assumeNotNull(hex(MD5( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 9f515f09a4a44..0000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,78 +0,0 @@ - - - create view _airbyte_test_normalization.pos_dedup_cdcx_ab3__dbt_tmp - - as ( - -with __dbt__cte__pos_dedup_cdcx_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - JSONExtractRaw(_airbyte_data, 'id') as id, - JSONExtractRaw(_airbyte_data, 'name') as name, - JSONExtractRaw(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - JSONExtractRaw(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - JSONExtractRaw(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - JSONExtractRaw(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 - -), __dbt__cte__pos_dedup_cdcx_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - accurateCastOrNull(id, ' - BIGINT -') as id, - nullif(accurateCastOrNull(trim(BOTH '"' from name), 'String'), 'null') as name, - accurateCastOrNull(_ab_cdc_lsn, ' - Float64 -') as _ab_cdc_lsn, - accurateCastOrNull(_ab_cdc_updated_at, ' - Float64 -') as _ab_cdc_updated_at, - accurateCastOrNull(_ab_cdc_deleted_at, ' - Float64 -') as _ab_cdc_deleted_at, - accurateCastOrNull(_ab_cdc_log_pos, ' - Float64 -') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from __dbt__cte__pos_dedup_cdcx_ab1 --- pos_dedup_cdcx -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - assumeNotNull(hex(MD5( - - toString(id) || '~' || - - - toString(name) || '~' || - - - toString(_ab_cdc_lsn) || '~' || - - - toString(_ab_cdc_updated_at) || '~' || - - - toString(_ab_cdc_deleted_at) || '~' || - - - toString(_ab_cdc_log_pos) - - ))) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from __dbt__cte__pos_dedup_cdcx_ab2 tmp --- pos_dedup_cdcx -where 1 = 1 - - ) \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 43c5b8ad9e18a..0000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,45 +0,0 @@ - - - create view _airbyte_test_normalization.renamed_dedup_cdc_excluded_ab3__dbt_tmp - - as ( - -with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - JSONExtractRaw(_airbyte_data, 'id') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - -), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - accurateCastOrNull(id, ' - BIGINT -') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from __dbt__cte__renamed_dedup_cdc_excluded_ab1 --- renamed_dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - assumeNotNull(hex(MD5( - - toString(id) - - ))) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ) \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index ad250a2de1969..5b9ee4b6b6820 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index b4921f53776b7..6f7e747a0699a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('dedup_cdc_excluded_ab1') }} select accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id, nullif(accurateCastOrNull(trim(BOTH '"' from name), '{{ dbt_utils.type_string() }}'), 'null') as name, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index bdfc716769aee..6e998ca141418 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 351ccad8f300e..ee41ee94585ee 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('dedup_exchange_rate_ab1') }} select accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id, nullif(accurateCastOrNull(trim(BOTH '"' from currency), '{{ dbt_utils.type_string() }}'), 'null') as currency, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 59f1c4bcfba0c..f9b9da32d25d1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index a48a14a7aecc3..49cb5ea4c759b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('exchange_rate_ab1') }} select accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id, nullif(accurateCastOrNull(trim(BOTH '"' from currency), '{{ dbt_utils.type_string() }}'), 'null') as currency, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index d6593d4eb8f1a..c45103fae85c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to build a hash column based on the values of this record +-- depends_on: {{ ref('exchange_rate_ab2') }} select {{ dbt_utils.surrogate_key([ 'id', diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 182bcd7dbfd4f..909b7bd2366b6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index 3769adf4d02e0..0b9192b2620a4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('pos_dedup_cdcx_ab1') }} select accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id, nullif(accurateCastOrNull(trim(BOTH '"' from name), '{{ dbt_utils.type_string() }}'), 'null') as name, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql index 4504a7bbffa32..a09668e69387e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, _airbyte_ab_id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql index 8b248db9590f7..2fd528509bc5a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('renamed_dedup_cdc_excluded_ab1') }} select accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id, _airbyte_ab_id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index eca9d38763a06..2c59c17df2e5b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -1,15 +1,17 @@ {{ config( unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", + post_hook = ['drop view _airbyte_test_normalization.dedup_cdc_excluded_stg'], tags = [ "top-level" ] ) }} +-- depends_on: ref('dedup_cdc_excluded_stg') with {% if is_incremental() %} new_data as ( -- retrieve incremental "new" data select * - from {{ ref('dedup_cdc_excluded_ab3') }} + from {{ ref('dedup_cdc_excluded_stg') }} -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} where 1 = 1 {{ incremental_clause('_airbyte_emitted_at') }} @@ -22,26 +24,30 @@ new_data_ids as ( ]) }} as _airbyte_unique_key from new_data ), +empty_new_data as ( + -- build an empty table to only keep the table's column types + select * from new_data where 1 = 0 +), previous_active_scd_data as ( -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes select - {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + {{ star_intersect(ref('dedup_cdc_excluded_stg'), this, from_alias='inc_data', intersect_alias='this_data') }} from {{ this }} as this_data -- make a join with new_data using primary key to filter active data that need to be updated only join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - --left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro on schema changes) + --left join empty_new_data as inc_data on this_data._airbyte_ab_id = inc_data._airbyte_ab_id where _airbyte_active_row = 1 ), input_data as ( - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + select {{ dbt_utils.star(ref('dedup_cdc_excluded_stg')) }} from new_data union all - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data + select {{ dbt_utils.star(ref('dedup_cdc_excluded_stg')) }} from previous_active_scd_data ), {% else %} input_data as ( select * - from {{ ref('dedup_cdc_excluded_ab3') }} + from {{ ref('dedup_cdc_excluded_stg') }} -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} ), {% endif %} @@ -50,8 +56,8 @@ input_data_with_active_row_num as ( row_number() over ( partition by id order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_lsn is null asc, + _ab_cdc_lsn desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc ) as _airbyte_active_row_num from input_data @@ -67,16 +73,16 @@ scd_data as ( _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - anyOrNull(_airbyte_emitted_at) over ( + _ab_cdc_lsn as _airbyte_start_at, + case when _airbyte_active_row_num = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + anyOrNull(_ab_cdc_lsn) over ( partition by id order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_lsn is null asc, + _ab_cdc_lsn desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING ) as _airbyte_end_at, - case when _airbyte_active_row_num = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_cdc_excluded_hashid @@ -87,8 +93,11 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, accurateCastOrNull(_ab_cdc_deleted_at, '{{ dbt_utils.type_string() }}'), accurateCastOrNull(_ab_cdc_updated_at, '{{ dbt_utils.type_string() }}') - order by _airbyte_ab_id + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at, accurateCastOrNull(_ab_cdc_deleted_at, '{{ dbt_utils.type_string() }}'), accurateCastOrNull(_ab_cdc_updated_at, '{{ dbt_utils.type_string() }}') + order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ '_airbyte_unique_key', diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 97757d03ce77d..98b9d2ea756d9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,15 +1,17 @@ {{ config( unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", + post_hook = ['drop view _airbyte_test_normalization.dedup_exchange_rate_stg'], tags = [ "top-level" ] ) }} +-- depends_on: ref('dedup_exchange_rate_stg') with {% if is_incremental() %} new_data as ( -- retrieve incremental "new" data select * - from {{ ref('dedup_exchange_rate_ab3') }} + from {{ ref('dedup_exchange_rate_stg') }} -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} where 1 = 1 {{ incremental_clause('_airbyte_emitted_at') }} @@ -24,26 +26,30 @@ new_data_ids as ( ]) }} as _airbyte_unique_key from new_data ), +empty_new_data as ( + -- build an empty table to only keep the table's column types + select * from new_data where 1 = 0 +), previous_active_scd_data as ( -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes select - {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + {{ star_intersect(ref('dedup_exchange_rate_stg'), this, from_alias='inc_data', intersect_alias='this_data') }} from {{ this }} as this_data -- make a join with new_data using primary key to filter active data that need to be updated only join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - --left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro on schema changes) + --left join empty_new_data as inc_data on this_data._airbyte_ab_id = inc_data._airbyte_ab_id where _airbyte_active_row = 1 ), input_data as ( - select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + select {{ dbt_utils.star(ref('dedup_exchange_rate_stg')) }} from new_data union all - select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data + select {{ dbt_utils.star(ref('dedup_exchange_rate_stg')) }} from previous_active_scd_data ), {% else %} input_data as ( select * - from {{ ref('dedup_exchange_rate_ab3') }} + from {{ ref('dedup_exchange_rate_stg') }} -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} ), {% endif %} @@ -75,15 +81,15 @@ scd_data as ( NZD, USD, date as _airbyte_start_at, + case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, anyOrNull(date) over ( partition by id, currency, cast(NZD as {{ dbt_utils.type_string() }}) order by date is null asc, date desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING ) as _airbyte_end_at, - case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid @@ -94,8 +100,11 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at + order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ '_airbyte_unique_key', diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index c0dcee2b2ccbb..0e66b7b0a12c6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -1,15 +1,17 @@ {{ config( unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", + post_hook = ['drop view _airbyte_test_normalization.renamed_dedup_cdc_excluded_stg'], tags = [ "top-level" ] ) }} +-- depends_on: ref('renamed_dedup_cdc_excluded_stg') with {% if is_incremental() %} new_data as ( -- retrieve incremental "new" data select * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + from {{ ref('renamed_dedup_cdc_excluded_stg') }} -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} where 1 = 1 {{ incremental_clause('_airbyte_emitted_at') }} @@ -22,26 +24,30 @@ new_data_ids as ( ]) }} as _airbyte_unique_key from new_data ), +empty_new_data as ( + -- build an empty table to only keep the table's column types + select * from new_data where 1 = 0 +), previous_active_scd_data as ( -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes select - {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + {{ star_intersect(ref('renamed_dedup_cdc_excluded_stg'), this, from_alias='inc_data', intersect_alias='this_data') }} from {{ this }} as this_data -- make a join with new_data using primary key to filter active data that need to be updated only join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - --left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro on schema changes) + --left join empty_new_data as inc_data on this_data._airbyte_ab_id = inc_data._airbyte_ab_id where _airbyte_active_row = 1 ), input_data as ( - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_stg')) }} from new_data union all - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_stg')) }} from previous_active_scd_data ), {% else %} input_data as ( select * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + from {{ ref('renamed_dedup_cdc_excluded_stg') }} -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} ), {% endif %} @@ -64,15 +70,15 @@ scd_data as ( ]) }} as _airbyte_unique_key, id, _airbyte_emitted_at as _airbyte_start_at, + case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, anyOrNull(_airbyte_emitted_at) over ( partition by id order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING ) as _airbyte_end_at, - case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_renamed_dedup_cdc_excluded_hashid @@ -83,8 +89,11 @@ dedup_data as ( -- we need to ensure de-duplicated rows for merge/update queries -- additionally, we generate a unique key for the scd table row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at + order by _airbyte_active_row desc, _airbyte_ab_id ) as _airbyte_row_num, {{ dbt_utils.surrogate_key([ '_airbyte_unique_key', diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index 086676f173719..6a6248e7cb6a8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -4,6 +4,7 @@ tags = [ "top-level" ] ) }} -- Final base SQL model +-- depends_on: {{ ref('dedup_cdc_excluded_scd') }} select _airbyte_unique_key, id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 3fb8238f2479b..180310a437ff6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -4,6 +4,7 @@ tags = [ "top-level" ] ) }} -- Final base SQL model +-- depends_on: {{ ref('dedup_exchange_rate_scd') }} select _airbyte_unique_key, id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index 8a8ff85f59024..d9f20813f833e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -4,6 +4,7 @@ tags = [ "top-level" ] ) }} -- Final base SQL model +-- depends_on: {{ ref('renamed_dedup_cdc_excluded_scd') }} select _airbyte_unique_key, id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql index c370f10264a3f..77fba32c34999 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -4,6 +4,7 @@ tags = [ "top-level" ] ) }} -- Final base SQL model +-- depends_on: {{ ref('exchange_rate_ab3') }} select id, currency, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 446204f691ebc..0000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql similarity index 92% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index b14a1fb639b1b..0b4900731039d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -4,6 +4,7 @@ tags = [ "top-level-intermediate" ] ) }} -- SQL model to build a hash column based on the values of this record +-- depends_on: {{ ref('dedup_exchange_rate_ab2') }} select {{ dbt_utils.surrogate_key([ 'id', diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index dbe0c313b238b..0000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 2356b929f1f38..0000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index 0eb15bc43e455..2ee3d293b8403 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -14,6 +14,7 @@ with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: test_normalization._airbyte_raw_exchange_rate select JSONExtractRaw(_airbyte_data, 'id') as id, JSONExtractRaw(_airbyte_data, 'currency') as currency, @@ -33,6 +34,7 @@ where 1 = 1 ), __dbt__cte__exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: __dbt__cte__exchange_rate_ab1 select accurateCastOrNull(id, ' BIGINT @@ -60,6 +62,7 @@ where 1 = 1 ), __dbt__cte__exchange_rate_ab3 as ( -- SQL model to build a hash column based on the values of this record +-- depends_on: __dbt__cte__exchange_rate_ab2 select assumeNotNull(hex(MD5( @@ -95,6 +98,7 @@ from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate where 1 = 1 )-- Final base SQL model +-- depends_on: __dbt__cte__exchange_rate_ab3 select id, currency, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index fe2bf632dbf20..0000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,71 +0,0 @@ - - - create view _airbyte_test_normalization.dedup_cdc_excluded_ab3__dbt_tmp - - as ( - -with __dbt__cte__dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - JSONExtractRaw(_airbyte_data, 'id') as id, - JSONExtractRaw(_airbyte_data, 'name') as name, - JSONExtractRaw(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - JSONExtractRaw(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - JSONExtractRaw(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - -), __dbt__cte__dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - accurateCastOrNull(id, ' - BIGINT -') as id, - nullif(accurateCastOrNull(trim(BOTH '"' from name), 'String'), 'null') as name, - accurateCastOrNull(_ab_cdc_lsn, ' - Float64 -') as _ab_cdc_lsn, - accurateCastOrNull(_ab_cdc_updated_at, ' - Float64 -') as _ab_cdc_updated_at, - accurateCastOrNull(_ab_cdc_deleted_at, ' - Float64 -') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from __dbt__cte__dedup_cdc_excluded_ab1 --- dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - assumeNotNull(hex(MD5( - - toString(id) || '~' || - - - toString(name) || '~' || - - - toString(_ab_cdc_lsn) || '~' || - - - toString(_ab_cdc_updated_at) || '~' || - - - toString(_ab_cdc_deleted_at) - - ))) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__dedup_cdc_excluded_ab2 tmp --- dedup_cdc_excluded -where 1 = 1 - - ) \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 28abd1a79a7f2..799af4ec78aba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,12 +1,13 @@ - create view _airbyte_test_normalization.dedup_exchange_rate_ab3__dbt_tmp + create view _airbyte_test_normalization.dedup_exchange_rate_stg__dbt_tmp as ( with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate select JSONExtractRaw(_airbyte_data, 'id') as id, JSONExtractRaw(_airbyte_data, 'currency') as currency, @@ -26,6 +27,7 @@ where 1 = 1 ), __dbt__cte__dedup_exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: __dbt__cte__dedup_exchange_rate_ab1 select accurateCastOrNull(id, ' BIGINT @@ -51,6 +53,7 @@ from __dbt__cte__dedup_exchange_rate_ab1 where 1 = 1 )-- SQL model to build a hash column based on the values of this record +-- depends_on: __dbt__cte__dedup_exchange_rate_ab2 select assumeNotNull(hex(MD5( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 9f515f09a4a44..0000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,78 +0,0 @@ - - - create view _airbyte_test_normalization.pos_dedup_cdcx_ab3__dbt_tmp - - as ( - -with __dbt__cte__pos_dedup_cdcx_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - JSONExtractRaw(_airbyte_data, 'id') as id, - JSONExtractRaw(_airbyte_data, 'name') as name, - JSONExtractRaw(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - JSONExtractRaw(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - JSONExtractRaw(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - JSONExtractRaw(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 - -), __dbt__cte__pos_dedup_cdcx_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - accurateCastOrNull(id, ' - BIGINT -') as id, - nullif(accurateCastOrNull(trim(BOTH '"' from name), 'String'), 'null') as name, - accurateCastOrNull(_ab_cdc_lsn, ' - Float64 -') as _ab_cdc_lsn, - accurateCastOrNull(_ab_cdc_updated_at, ' - Float64 -') as _ab_cdc_updated_at, - accurateCastOrNull(_ab_cdc_deleted_at, ' - Float64 -') as _ab_cdc_deleted_at, - accurateCastOrNull(_ab_cdc_log_pos, ' - Float64 -') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from __dbt__cte__pos_dedup_cdcx_ab1 --- pos_dedup_cdcx -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - assumeNotNull(hex(MD5( - - toString(id) || '~' || - - - toString(name) || '~' || - - - toString(_ab_cdc_lsn) || '~' || - - - toString(_ab_cdc_updated_at) || '~' || - - - toString(_ab_cdc_deleted_at) || '~' || - - - toString(_ab_cdc_log_pos) - - ))) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from __dbt__cte__pos_dedup_cdcx_ab2 tmp --- pos_dedup_cdcx -where 1 = 1 - - ) \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 43c5b8ad9e18a..0000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,45 +0,0 @@ - - - create view _airbyte_test_normalization.renamed_dedup_cdc_excluded_ab3__dbt_tmp - - as ( - -with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - JSONExtractRaw(_airbyte_data, 'id') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - -), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - accurateCastOrNull(id, ' - BIGINT -') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from __dbt__cte__renamed_dedup_cdc_excluded_ab1 --- renamed_dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - assumeNotNull(hex(MD5( - - toString(id) - - ))) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ) \ No newline at end of file From f536e48a8fd64110e379b8f067ae8878b69d7567 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Wed, 5 Jan 2022 12:56:57 +0100 Subject: [PATCH 13/16] Regen file --- .../dbt-project-template-clickhouse/dbt_project.yml | 3 --- .../dbt-project-template-mssql/dbt_project.yml | 1 - .../dbt-project-template-mysql/dbt_project.yml | 1 - .../dbt-project-template-oracle/dbt_project.yml | 1 - .../base-normalization/dbt-project-template/dbt_project.yml | 3 --- .../bigquery/test_nested_streams/dbt_project.yml | 3 --- .../bigquery/test_simple_streams/dbt_project.yml | 3 --- .../bigquery/test_simple_streams/first_dbt_project.yml | 3 --- .../clickhouse/test_simple_streams/dbt_project.yml | 3 --- .../mssql/test_nested_streams/dbt_project.yml | 1 - .../mssql/test_simple_streams/dbt_project.yml | 1 - .../mysql/test_nested_streams/dbt_project.yml | 1 - .../mysql/test_simple_streams/dbt_project.yml | 1 - .../oracle/test_simple_streams/dbt_project.yml | 1 - .../postgres/test_nested_streams/dbt_project.yml | 3 --- .../postgres/test_simple_streams/dbt_project.yml | 3 --- .../postgres/test_simple_streams/first_dbt_project.yml | 3 --- .../redshift/test_nested_streams/dbt_project.yml | 3 --- .../redshift/test_simple_streams/dbt_project.yml | 3 --- .../redshift/test_simple_streams/first_dbt_project.yml | 3 --- .../snowflake/test_nested_streams/dbt_project.yml | 3 --- .../snowflake/test_simple_streams/dbt_project.yml | 3 --- 22 files changed, 50 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-clickhouse/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-clickhouse/dbt_project.yml index ae74081e6a4e9..37eca7fcd73f3 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-clickhouse/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-clickhouse/dbt_project.yml @@ -62,6 +62,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "clickhouse" \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-mssql/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-mssql/dbt_project.yml index 111260bb1df7d..8c7494fdc58f5 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-mssql/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-mssql/dbt_project.yml @@ -59,4 +59,3 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] - destination: "mssql" diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml index dcf0bcf8e1885..b03cee8fe9305 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml @@ -61,4 +61,3 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] - destination: "myssql" diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-oracle/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-oracle/dbt_project.yml index 4ed6a85b82a44..0ded2a42d60e9 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-oracle/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-oracle/dbt_project.yml @@ -59,4 +59,3 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] - destination: "oracle" diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template/dbt_project.yml index 7ad0c2ea3ed5f..9ad8158759001 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml index 7ad0c2ea3ed5f..9ad8158759001 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml index 7568c2e4aea40..522b1e595e8ac 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml index 7ad0c2ea3ed5f..9ad8158759001 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml index ae74081e6a4e9..37eca7fcd73f3 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml @@ -62,6 +62,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "clickhouse" \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml index 111260bb1df7d..8c7494fdc58f5 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml @@ -59,4 +59,3 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] - destination: "mssql" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml index 111260bb1df7d..8c7494fdc58f5 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml @@ -59,4 +59,3 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] - destination: "mssql" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml index dcf0bcf8e1885..b03cee8fe9305 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml @@ -61,4 +61,3 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] - destination: "myssql" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml index dcf0bcf8e1885..b03cee8fe9305 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml @@ -61,4 +61,3 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] - destination: "myssql" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml index 4ed6a85b82a44..0ded2a42d60e9 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml @@ -59,4 +59,3 @@ models: vars: dbt_utils_dispatch_list: ['airbyte_utils'] - destination: "oracle" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml index 7ad0c2ea3ed5f..9ad8158759001 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml index 7568c2e4aea40..522b1e595e8ac 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml index 7ad0c2ea3ed5f..9ad8158759001 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml index 7ad0c2ea3ed5f..9ad8158759001 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml index 7568c2e4aea40..522b1e595e8ac 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml index 7ad0c2ea3ed5f..9ad8158759001 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml index 7ad0c2ea3ed5f..9ad8158759001 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml index 7ad0c2ea3ed5f..9ad8158759001 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml @@ -61,6 +61,3 @@ models: dispatch: - macro_namespace: dbt_utils search_order: ['airbyte_utils', 'dbt_utils'] - -vars: - destination: "default" From 7b303bb8840a2e3cd889e57dd1cfaa9a7260c46f Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Wed, 5 Jan 2022 14:54:59 +0100 Subject: [PATCH 14/16] Re-indent columns --- ..._columns_resulting_into_long_names_scd.sql | 12 +- ..._columns_resulting_into_long_names_scd.sql | 14 +- .../dedup_exchange_rate_scd.sql | 32 ++--- .../dedup_exchange_rate_scd.sql | 38 +++--- .../dedup_exchange_rate_scd.sql | 38 +++--- .../dedup_cdc_excluded_scd.sql | 29 ++-- .../dedup_exchange_rate_scd.sql | 35 +++-- .../renamed_dedup_cdc_excluded_scd.sql | 21 +-- .../renamed_dedup_cdc_excluded.sql | 1 + .../renamed_dedup_cdc_excluded_ab1.sql | 1 + .../renamed_dedup_cdc_excluded_ab2.sql | 1 + .../dedup_cdc_excluded_scd.sql | 31 ++--- .../dedup_exchange_rate_scd.sql | 41 +++--- .../renamed_dedup_cdc_excluded_scd.sql | 23 ++-- .../renamed_dedup_cdc_excluded.sql | 1 + .../renamed_dedup_cdc_excluded_scd.sql | 4 +- .../renamed_dedup_cdc_excluded.sql | 4 +- ...eam_with_co__lting_into_long_names_scd.sql | 14 +- ...eam_with_co__lting_into_long_names_scd.sql | 16 +-- .../dedup_exchange_rate_scd.sql | 34 +++-- .../dedup_exchange_rate_scd.sql | 40 +++--- ..._stream_with_co_1g_into_long_names_scd.sql | 12 +- ..._stream_with_co_1g_into_long_names_scd.sql | 14 +- ..._stream_with_co_1g_into_long_names_scd.sql | 12 +- .../dedup_exchange_rate_scd.sql | 32 ++--- .../dedup_exchange_rate_scd.sql | 38 +++--- .../dedup_exchange_rate_scd.sql | 32 ++--- .../dedup_exchange_rate_scd.sql | 34 +++-- .../dedup_exchange_rate_scd.sql | 40 +++--- .../dedup_exchange_rate_scd.sql | 34 +++-- ...ream_with_c__lting_into_long_names_scd.sql | 12 +- .../some_stream_that_was_empty_scd.sql | 8 +- ...ream_with_c__lting_into_long_names_scd.sql | 14 +- .../some_stream_that_was_empty_scd.sql | 10 +- .../dedup_cdc_excluded_scd.sql | 26 ++-- .../dedup_exchange_rate_scd.sql | 32 ++--- .../test_normalization/pos_dedup_cdcx_scd.sql | 44 +++--- .../renamed_dedup_cdc_excluded_scd.sql | 18 +-- .../renamed_dedup_cdc_excluded.sql | 1 + .../renamed_dedup_cdc_excluded_stg.sql | 6 + .../renamed_dedup_cdc_excluded_ab1.sql | 1 + .../renamed_dedup_cdc_excluded_ab2.sql | 1 + .../dedup_cdc_excluded_scd.sql | 28 ++-- .../dedup_exchange_rate_scd.sql | 38 +++--- .../test_normalization/pos_dedup_cdcx_scd.sql | 46 ++++--- .../renamed_dedup_cdc_excluded_scd.sql | 20 +-- .../renamed_dedup_cdc_excluded.sql | 1 + .../renamed_dedup_cdc_excluded_stg.sql | 1 + .../dedup_exchange_rate_scd.sql | 38 +++--- .../renamed_dedup_cdc_excluded_scd.sql | 40 +++--- .../renamed_dedup_cdc_excluded_scd.sql | 4 +- .../renamed_dedup_cdc_excluded.sql | 4 +- .../renamed_dedup_cdc_excluded_stg.sql | 4 +- .../renamed_dedup_cdc_excluded_scd.sql | 4 +- .../renamed_dedup_cdc_excluded.sql | 4 +- .../renamed_dedup_cdc_excluded_stg.sql | 4 +- ..._columns_resulting_into_long_names_scd.sql | 12 +- ..._columns_resulting_into_long_names_scd.sql | 14 +- .../dedup_exchange_rate_scd.sql | 32 ++--- .../dedup_exchange_rate_scd.sql | 38 +++--- .../dedup_exchange_rate_scd.sql | 38 +++--- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 12 +- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 14 +- .../DEDUP_EXCHANGE_RATE_SCD.sql | 32 ++--- .../DEDUP_EXCHANGE_RATE_SCD.sql | 38 +++--- .../data_input/catalog.json | 3 + .../transform_catalog/stream_processor.py | 127 +++++++++--------- 67 files changed, 739 insertions(+), 708 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 60a896682fffc..59cf6d3a78044 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -26,9 +26,9 @@ scd_data as ( ), '')) as string ))) as _airbyte_unique_key, - id, - date, - `partition`, + id, + date, + `partition`, date as _airbyte_start_at, lag(date) over ( partition by id @@ -75,9 +75,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - date, - `partition`, + id, + date, + `partition`, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 1dd492613c5a7..a1d766dd11035 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -57,11 +57,11 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', + 'id', ]) }} as _airbyte_unique_key, - id, - date, - {{ adapter.quote('partition') }}, + id, + date, + {{ adapter.quote('partition') }}, date as _airbyte_start_at, lag(date) over ( partition by id @@ -104,9 +104,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - date, - {{ adapter.quote('partition') }}, + id, + date, + {{ adapter.quote('partition') }}, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index da4e350a2da76..d7fd59df15b5d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -30,14 +30,14 @@ scd_data as ( ), '')) as string ))) as _airbyte_unique_key, - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, date as _airbyte_start_at, lag(date) over ( partition by id, currency, cast(NZD as @@ -88,14 +88,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index d31c177ae671a..c2edf0afe7961 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -59,18 +59,18 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'NZD', + 'id', + 'currency', + 'NZD', ]) }} as _airbyte_unique_key, - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, date as _airbyte_start_at, lag(date) over ( partition by id, currency, cast(NZD as {{ dbt_utils.type_string() }}) @@ -113,14 +113,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 1284489491f70..cc6694836bb6a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -59,18 +59,18 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'NZD', + 'id', + 'currency', + 'NZD', ]) }} as _airbyte_unique_key, - id, - currency, - new_column, - date, - timestamp_col, - HKD_special___characters, - NZD, - USD, + id, + currency, + new_column, + date, + timestamp_col, + HKD_special___characters, + NZD, + USD, date as _airbyte_start_at, lag(date) over ( partition by cast(id as {{ dbt_utils.type_string() }}), currency, cast(NZD as {{ dbt_utils.type_string() }}) @@ -113,14 +113,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - new_column, - date, - timestamp_col, - HKD_special___characters, - NZD, - USD, + id, + currency, + new_column, + date, + timestamp_col, + HKD_special___characters, + NZD, + USD, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index ca78582b180d9..c0ffb324a53dc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -28,7 +28,8 @@ input_data_with_active_row_num as ( order by _ab_cdc_lsn is null asc, _ab_cdc_lsn desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc ) as _airbyte_active_row_num from input_data ), @@ -40,11 +41,11 @@ scd_data as ( toString(id) ))) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, _ab_cdc_lsn as _airbyte_start_at, case when _airbyte_active_row_num = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, anyOrNull(_ab_cdc_lsn) over ( @@ -52,9 +53,9 @@ scd_data as ( order by _ab_cdc_lsn is null asc, _ab_cdc_lsn desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING - ) as _airbyte_end_at, + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_cdc_excluded_hashid @@ -94,11 +95,11 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 93e86c0b32d3b..f30017d3cd5e5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -46,14 +46,14 @@ scd_data as ( toString(NZD) ))) as _airbyte_unique_key, - id, - currency, - date, - timestamp_col, - "HKD@spéçiäl & characters", - HKD_special___characters, - NZD, - USD, + id, + currency, + date, + timestamp_col, + "HKD@spéçiäl & characters", + HKD_special___characters, + NZD, + USD, date as _airbyte_start_at, case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, anyOrNull(date) over ( @@ -62,8 +62,7 @@ scd_data as ( date is null asc, date desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING - ) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid @@ -97,14 +96,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - date, - timestamp_col, - "HKD@spéçiäl & characters", - HKD_special___characters, - NZD, - USD, + id, + currency, + date, + timestamp_col, + "HKD@spéçiäl & characters", + HKD_special___characters, + NZD, + USD, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index d96c53f97da05..04d74982d7b2c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -26,8 +26,8 @@ input_data_with_active_row_num as ( row_number() over ( partition by id order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, _airbyte_emitted_at desc ) as _airbyte_active_row_num from input_data @@ -40,17 +40,17 @@ scd_data as ( toString(id) ))) as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, + id, + _ab_cdc_updated_at, + _ab_cdc_updated_at as _airbyte_start_at, case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, - anyOrNull(_airbyte_emitted_at) over ( + anyOrNull(_ab_cdc_updated_at) over ( partition by id order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING - ) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_renamed_dedup_cdc_excluded_hashid @@ -84,7 +84,8 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, + id, + _ab_cdc_updated_at, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index b16b5361120f0..9966d52012aec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -17,6 +17,7 @@ select _airbyte_unique_key, id, + _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql index a09668e69387e..88a3674f694b3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -7,6 +7,7 @@ -- depends_on: {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql index 2fd528509bc5a..b192f4915e98f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -7,6 +7,7 @@ -- depends_on: {{ ref('renamed_dedup_cdc_excluded_ab1') }} select accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id, + accurateCastOrNull(_ab_cdc_updated_at, '{{ dbt_utils.type_float() }}') as _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index 2c59c17df2e5b..c9b7736da2845 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -58,7 +58,8 @@ input_data_with_active_row_num as ( order by _ab_cdc_lsn is null asc, _ab_cdc_lsn desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc ) as _airbyte_active_row_num from input_data ), @@ -66,13 +67,13 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', + 'id', ]) }} as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, _ab_cdc_lsn as _airbyte_start_at, case when _airbyte_active_row_num = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, anyOrNull(_ab_cdc_lsn) over ( @@ -80,9 +81,9 @@ scd_data as ( order by _ab_cdc_lsn is null asc, _ab_cdc_lsn desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING - ) as _airbyte_end_at, + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_cdc_excluded_hashid @@ -110,11 +111,11 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 98b9d2ea756d9..e5e0e0f749ffe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -68,18 +68,18 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'NZD', + 'id', + 'currency', + 'NZD', ]) }} as _airbyte_unique_key, - id, - currency, - date, - timestamp_col, - {{ quote('HKD@spéçiäl & characters') }}, - HKD_special___characters, - NZD, - USD, + id, + currency, + date, + timestamp_col, + {{ quote('HKD@spéçiäl & characters') }}, + HKD_special___characters, + NZD, + USD, date as _airbyte_start_at, case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, anyOrNull(date) over ( @@ -88,8 +88,7 @@ scd_data as ( date is null asc, date desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING - ) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid @@ -117,14 +116,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - date, - timestamp_col, - {{ quote('HKD@spéçiäl & characters') }}, - HKD_special___characters, - NZD, - USD, + id, + currency, + date, + timestamp_col, + {{ quote('HKD@spéçiäl & characters') }}, + HKD_special___characters, + NZD, + USD, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 0e66b7b0a12c6..c79c9d716682e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -56,8 +56,8 @@ input_data_with_active_row_num as ( row_number() over ( partition by id order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, _airbyte_emitted_at desc ) as _airbyte_active_row_num from input_data @@ -66,19 +66,19 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', + 'id', ]) }} as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, + id, + _ab_cdc_updated_at, + _ab_cdc_updated_at as _airbyte_start_at, case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row, - anyOrNull(_airbyte_emitted_at) over ( + anyOrNull(_ab_cdc_updated_at) over ( partition by id order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING - ) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_renamed_dedup_cdc_excluded_hashid @@ -106,7 +106,8 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, + id, + _ab_cdc_updated_at, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index d9f20813f833e..1b9cead2c4958 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -8,6 +8,7 @@ select _airbyte_unique_key, id, + _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 0fc967c7e00a1..f96e982626f6a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -1,5 +1,5 @@ - insert into test_normalization.renamed_dedup_cdc_excluded_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + insert into test_normalization.renamed_dedup_cdc_excluded_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" from renamed_dedup_cdc_excluded_scd__dbt_tmp \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index 3ee365f07d589..4ff849492d8ab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -1,5 +1,5 @@ - insert into test_normalization.renamed_dedup_cdc_excluded ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") - select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + insert into test_normalization.renamed_dedup_cdc_excluded ("_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + select "_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" from renamed_dedup_cdc_excluded__dbt_tmp \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql index 904d4721cbcd3..113bf3c06d961 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -39,21 +39,19 @@ scd_data as ( concat(concat(coalesce(cast(id as VARCHAR(max)), ''''),''''), '''') as VARCHAR(max)), '''')), 2) as _airbyte_unique_key, - id, - "date", - "partition", + id, + "date", + "partition", "date" as _airbyte_start_at, lag("date") over ( partition by id order by - "date" desc, "date" desc, _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by id order by - "date" desc, "date" desc, _airbyte_emitted_at desc ) = 1 then 1 else 0 end as _airbyte_active_row, @@ -88,9 +86,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - "date", - "partition", + id, + "date", + "partition", _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql index 32f5b97aaff50..b04acfdabc525 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -55,23 +55,21 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', + 'id', ]) }} as _airbyte_unique_key, - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, {{ adapter.quote('date') }} as _airbyte_start_at, lag({{ adapter.quote('date') }}) over ( partition by id order by - {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by id order by - {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) = 1 then 1 else 0 end as _airbyte_active_row, @@ -102,9 +100,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 198c37bf1797f..ea79968965206 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -41,20 +41,19 @@ scd_data as ( VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as VARCHAR(max)), ''''),''''), '''') as VARCHAR(max)), '''')), 2) as _airbyte_unique_key, - id, - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, "date" as _airbyte_start_at, lag("date") over ( partition by id, currency, cast(nzd as VARCHAR(max)) order by - "date" desc, "date" desc, _airbyte_emitted_at desc ) as _airbyte_end_at, @@ -62,7 +61,6 @@ scd_data as ( partition by id, currency, cast(nzd as VARCHAR(max)) order by - "date" desc, "date" desc, _airbyte_emitted_at desc ) = 1 then 1 else 0 end as _airbyte_active_row, @@ -97,14 +95,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 48fa805a9ce7f..fcc681aa95ba8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -57,30 +57,28 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'nzd', + 'id', + 'currency', + 'nzd', ]) }} as _airbyte_unique_key, - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, {{ adapter.quote('date') }} as _airbyte_start_at, lag({{ adapter.quote('date') }}) over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by - {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by - {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) = 1 then 1 else 0 end as _airbyte_active_row, @@ -111,14 +109,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index 51184a5a90a23..e5f3e4859deba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -17,9 +17,9 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, - id, - `date`, - `partition`, + id, + `date`, + `partition`, `date` as _airbyte_start_at, lag(`date`) over ( partition by id @@ -58,9 +58,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - `date`, - `partition`, + id, + `date`, + `partition`, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index 9f7abb50d9d30..d098146930d0c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -55,11 +55,11 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', + 'id', ]) }} as _airbyte_unique_key, - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, {{ adapter.quote('date') }} as _airbyte_start_at, lag({{ adapter.quote('date') }}) over ( partition by id @@ -102,9 +102,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index 51184a5a90a23..e5f3e4859deba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -17,9 +17,9 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, - id, - `date`, - `partition`, + id, + `date`, + `partition`, `date` as _airbyte_start_at, lag(`date`) over ( partition by id @@ -58,9 +58,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - `date`, - `partition`, + id, + `date`, + `partition`, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 30eb801075502..59d722cb4f381 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -17,14 +17,14 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(nzd as char), '')) as char)) as _airbyte_unique_key, - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, `date` as _airbyte_start_at, lag(`date`) over ( partition by id, currency, cast(nzd as char) @@ -63,14 +63,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index ed12ca056e793..309ac4c903fe5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -57,18 +57,18 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'nzd', + 'id', + 'currency', + 'nzd', ]) }} as _airbyte_unique_key, - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, {{ adapter.quote('date') }} as _airbyte_start_at, lag({{ adapter.quote('date') }}) over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) @@ -111,14 +111,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 30eb801075502..59d722cb4f381 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -17,14 +17,14 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(nzd as char), '')) as char)) as _airbyte_unique_key, - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, `date` as _airbyte_start_at, lag(`date`) over ( partition by id, currency, cast(nzd as char) @@ -63,14 +63,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index db458b497ca2f..cfd186b006ae3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -27,27 +27,25 @@ scd_data as ( nzd ) as "_AIRBYTE_UNIQUE_KEY", - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, "DATE" as "_AIRBYTE_START_AT", lag("DATE") over ( partition by id, currency, cast(nzd as varchar2(4000)) order by "DATE" desc nulls last, - "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) as "_AIRBYTE_END_AT", case when row_number() over ( partition by id, currency, cast(nzd as varchar2(4000)) order by "DATE" desc nulls last, - "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) = 1 then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", @@ -83,14 +81,14 @@ dedup_data as ( select "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index bd2717a7a654d..8e8364a7b5072 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -57,31 +57,29 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'nzd', + 'id', + 'currency', + 'nzd', ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - id, - currency, - {{ quote('DATE') }}, - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, + id, + currency, + {{ quote('DATE') }}, + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, {{ quote('DATE') }} as {{ quote('_AIRBYTE_START_AT') }}, lag({{ quote('DATE') }}) over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by {{ quote('DATE') }} desc nulls last, - {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc ) as {{ quote('_AIRBYTE_END_AT') }}, case when row_number() over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by {{ quote('DATE') }} desc nulls last, - {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc ) = 1 then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, {{ quote('_AIRBYTE_AB_ID') }}, @@ -111,14 +109,14 @@ dedup_data as ( select {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, - id, - currency, - {{ quote('DATE') }}, - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, + id, + currency, + {{ quote('DATE') }}, + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_END_AT') }}, {{ quote('_AIRBYTE_ACTIVE_ROW') }}, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index db458b497ca2f..cfd186b006ae3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -27,27 +27,25 @@ scd_data as ( nzd ) as "_AIRBYTE_UNIQUE_KEY", - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, "DATE" as "_AIRBYTE_START_AT", lag("DATE") over ( partition by id, currency, cast(nzd as varchar2(4000)) order by "DATE" desc nulls last, - "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) as "_AIRBYTE_END_AT", case when row_number() over ( partition by id, currency, cast(nzd as varchar2(4000)) order by "DATE" desc nulls last, - "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) = 1 then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", @@ -83,14 +81,14 @@ dedup_data as ( select "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql index 14c58318d53e1..b5d7f740ba6eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -21,9 +21,9 @@ scd_data as ( ), '') as varchar )) as _airbyte_unique_key, - "id", - "date", - "partition", + "id", + "date", + "partition", "date" as _airbyte_start_at, lag("date") over ( partition by "id" @@ -70,9 +70,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - "id", - "date", - "partition", + "id", + "date", + "partition", _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql index 8ad5609bf2134..53ef64cb928a8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql @@ -21,8 +21,8 @@ scd_data as ( ), '') as varchar )) as _airbyte_unique_key, - "id", - "date", + "id", + "date", "date" as _airbyte_start_at, lag("date") over ( partition by "id" @@ -69,8 +69,8 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - "id", - "date", + "id", + "date", _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql index ee15d789a7d4d..8772de10b5e74 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -56,11 +56,11 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - adapter.quote('id'), + adapter.quote('id'), ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, {{ adapter.quote('date') }} as _airbyte_start_at, lag({{ adapter.quote('date') }}) over ( partition by {{ adapter.quote('id') }} @@ -103,9 +103,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - {{ adapter.quote('id') }}, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql index 3194631fa65f9..1eba7ba7bd0ba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql @@ -56,10 +56,10 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - adapter.quote('id'), + adapter.quote('id'), ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - {{ adapter.quote('date') }}, + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, {{ adapter.quote('date') }} as _airbyte_start_at, lag({{ adapter.quote('date') }}) over ( partition by {{ adapter.quote('id') }} @@ -102,8 +102,8 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - {{ adapter.quote('id') }}, - {{ adapter.quote('date') }}, + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index c0c8a4a752d0b..c9c2e087d956b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -21,25 +21,27 @@ scd_data as ( ), '') as varchar )) as _airbyte_unique_key, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, _ab_cdc_lsn as _airbyte_start_at, lag(_ab_cdc_lsn) over ( partition by "id" order by _ab_cdc_lsn is null asc, _ab_cdc_lsn desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by "id" order by _ab_cdc_lsn is null asc, _ab_cdc_lsn desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, @@ -80,11 +82,11 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index a92502150e8b4..3db3150ff2766 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -25,14 +25,14 @@ scd_data as ( ), '') as varchar )) as _airbyte_unique_key, - "id", - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, + "id", + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, "date" as _airbyte_start_at, lag("date") over ( partition by "id", currency, cast(nzd as @@ -83,14 +83,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - "id", - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, + "id", + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql index a4541da8b27bd..438b303238b5e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -21,26 +21,30 @@ scd_data as ( ), '') as varchar )) as _airbyte_unique_key, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _ab_cdc_updated_at as _airbyte_start_at, + lag(_ab_cdc_updated_at) over ( partition by "id" order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, + _ab_cdc_updated_at desc, + _ab_cdc_log_pos desc, + _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by "id" order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, + _ab_cdc_updated_at desc, + _ab_cdc_log_pos desc, + _airbyte_emitted_at desc ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, @@ -85,12 +89,12 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 0f7b4b8de97ac..414ed447cc0b5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -21,20 +21,21 @@ scd_data as ( ), '') as varchar )) as _airbyte_unique_key, - "id", - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( + "id", + _ab_cdc_updated_at, + _ab_cdc_updated_at as _airbyte_start_at, + lag(_ab_cdc_updated_at) over ( partition by "id" order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by "id" order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, _airbyte_emitted_at desc ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, @@ -68,7 +69,8 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - "id", + "id", + _ab_cdc_updated_at, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index d2a4347421f07..36303d71ef60e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -9,6 +9,7 @@ select _airbyte_unique_key, "id", + _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql index a31227240b16b..8fb3cb3a5c344 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql @@ -10,6 +10,7 @@ with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( -- depends_on: "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded select jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at @@ -25,6 +26,9 @@ select cast("id" as bigint ) as "id", + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at @@ -37,6 +41,8 @@ where 1 = 1 select md5(cast(coalesce(cast("id" as varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar ), '') as varchar )) as _airbyte_renamed_dedup_cdc_excluded_hashid, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql index 332bcdac0d8ab..e75261bd70a4e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -8,6 +8,7 @@ -- depends_on: {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, + {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql index 54dbe8bb35287..f7a91a73a73cb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -8,6 +8,7 @@ -- depends_on: {{ ref('renamed_dedup_cdc_excluded_ab1') }} select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, + cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index 50f9ec0632708..98325193a5f00 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -56,27 +56,29 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - adapter.quote('id'), + adapter.quote('id'), ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, _ab_cdc_lsn as _airbyte_start_at, lag(_ab_cdc_lsn) over ( partition by {{ adapter.quote('id') }} order by _ab_cdc_lsn is null asc, _ab_cdc_lsn desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by {{ adapter.quote('id') }} order by _ab_cdc_lsn is null asc, _ab_cdc_lsn desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, @@ -105,11 +107,11 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index d1e9d599e25f3..87453dc261145 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -58,18 +58,18 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - 'currency', - 'nzd', + adapter.quote('id'), + 'currency', + 'nzd', ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, + {{ adapter.quote('id') }}, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, {{ adapter.quote('date') }} as _airbyte_start_at, lag({{ adapter.quote('date') }}) over ( partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) @@ -112,14 +112,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - {{ adapter.quote('id') }}, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, + {{ adapter.quote('id') }}, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql index 213f78fd53691..36ce51399a3f3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -56,28 +56,32 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - adapter.quote('id'), + adapter.quote('id'), ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _ab_cdc_updated_at as _airbyte_start_at, + lag(_ab_cdc_updated_at) over ( partition by {{ adapter.quote('id') }} order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, + _ab_cdc_updated_at desc, + _ab_cdc_log_pos desc, + _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by {{ adapter.quote('id') }} order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, + _ab_cdc_updated_at desc, + _ab_cdc_log_pos desc, + _airbyte_emitted_at desc ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, @@ -106,12 +110,12 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 5c9e9fa82cf35..0c50939426f79 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -56,22 +56,23 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - adapter.quote('id'), + adapter.quote('id'), ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( + {{ adapter.quote('id') }}, + _ab_cdc_updated_at, + _ab_cdc_updated_at as _airbyte_start_at, + lag(_ab_cdc_updated_at) over ( partition by {{ adapter.quote('id') }} order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by {{ adapter.quote('id') }} order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, _airbyte_emitted_at desc ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, @@ -101,7 +102,8 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - {{ adapter.quote('id') }}, + {{ adapter.quote('id') }}, + _ab_cdc_updated_at, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index 9b58ab7c73f2f..ca5093eb3e17e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -9,6 +9,7 @@ select _airbyte_unique_key, {{ adapter.quote('id') }}, + _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql index 8b713b1e15b3d..be9bbfcd86758 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql @@ -9,6 +9,7 @@ select {{ dbt_utils.surrogate_key([ adapter.quote('id'), + '_ab_cdc_updated_at', ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, tmp.* from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 8a71da096568d..5db2e106d7ddf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -58,18 +58,18 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - 'currency', - 'nzd', + adapter.quote('id'), + 'currency', + 'nzd', ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - currency, - new_column, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - nzd, - usd, + {{ adapter.quote('id') }}, + currency, + new_column, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + nzd, + usd, {{ adapter.quote('date') }} as _airbyte_start_at, lag({{ adapter.quote('date') }}) over ( partition by cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}), currency, cast(nzd as {{ dbt_utils.type_string() }}) @@ -112,14 +112,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - {{ adapter.quote('id') }}, - currency, - new_column, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - nzd, - usd, + {{ adapter.quote('id') }}, + currency, + new_column, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index efc5b2c1671b6..3803571720588 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -56,27 +56,29 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - adapter.quote('id'), + adapter.quote('id'), ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_updated_at as _airbyte_start_at, + lag(_ab_cdc_updated_at) over ( partition by {{ adapter.quote('id') }} order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc ) as _airbyte_end_at, case when row_number() over ( partition by {{ adapter.quote('id') }} order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc + _ab_cdc_updated_at is null asc, + _ab_cdc_updated_at desc, + _ab_cdc_updated_at desc, + _airbyte_emitted_at desc ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, @@ -105,11 +107,11 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 5536e95b30750..3ea9e1c724fb6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -6,9 +6,9 @@ from "renamed_dedup_cdc_excluded_scd__dbt_tmp" ); - insert into "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" from "renamed_dedup_cdc_excluded_scd__dbt_tmp" ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index a5cc40567b2cb..3fec1976ed9d0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -6,9 +6,9 @@ from "renamed_dedup_cdc_excluded__dbt_tmp" ); - insert into "postgres".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") ( - select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + select "_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" from "renamed_dedup_cdc_excluded__dbt_tmp" ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql index 502e7141b3e86..5f39006c6490b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql @@ -6,9 +6,9 @@ from "renamed_dedup_cdc_excluded_stg__dbt_tmp" ); - insert into "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_stg" ("_airbyte_renamed_dedup_cdc_excluded_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at") + insert into "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_stg" ("_airbyte_renamed_dedup_cdc_excluded_hashid", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at") ( - select "_airbyte_renamed_dedup_cdc_excluded_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at" + select "_airbyte_renamed_dedup_cdc_excluded_hashid", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at" from "renamed_dedup_cdc_excluded_stg__dbt_tmp" ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 7693af7ef2e63..dfe10c6da794d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -6,9 +6,9 @@ from "renamed_dedup_cdc_excluded_scd__dbt_tmp" ); - insert into "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_deleted_at") ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_deleted_at" from "renamed_dedup_cdc_excluded_scd__dbt_tmp" ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index c8edd1056dd5d..c1d1c310179d3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -6,9 +6,9 @@ from "renamed_dedup_cdc_excluded__dbt_tmp" ); - insert into "postgres".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_deleted_at") ( - select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + select "_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_deleted_at" from "renamed_dedup_cdc_excluded__dbt_tmp" ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql index 62a33963d7a20..55db812277ae0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_stg.sql @@ -6,9 +6,9 @@ from "renamed_dedup_cdc_excluded_stg__dbt_tmp" ); - insert into "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_stg" ("_airbyte_renamed_dedup_cdc_excluded_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + insert into "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_stg" ("_airbyte_renamed_dedup_cdc_excluded_hashid", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "name", "_ab_cdc_lsn", "_ab_cdc_deleted_at") ( - select "_airbyte_renamed_dedup_cdc_excluded_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + select "_airbyte_renamed_dedup_cdc_excluded_hashid", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "name", "_ab_cdc_lsn", "_ab_cdc_deleted_at" from "renamed_dedup_cdc_excluded_stg__dbt_tmp" ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 0ed248505d323..0f50d29444876 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -21,9 +21,9 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, - id, - date, - "partition", + id, + date, + "partition", date as _airbyte_start_at, lag(date) over ( partition by id @@ -62,9 +62,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - date, - "partition", + id, + date, + "partition", _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 695f5595a996e..1e30bf57e4595 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -56,11 +56,11 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', + 'id', ]) }} as _airbyte_unique_key, - id, - date, - {{ adapter.quote('partition') }}, + id, + date, + {{ adapter.quote('partition') }}, date as _airbyte_start_at, lag(date) over ( partition by id @@ -103,9 +103,9 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - date, - {{ adapter.quote('partition') }}, + id, + date, + {{ adapter.quote('partition') }}, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 69cafeae2f223..e2fb4b8024b06 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -21,14 +21,14 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') as varchar)) as _airbyte_unique_key, - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, date as _airbyte_start_at, lag(date) over ( partition by id, currency, cast(nzd as varchar) @@ -67,14 +67,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index fbbbd04bae94f..81b85e492cd51 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -58,18 +58,18 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'nzd', + 'id', + 'currency', + 'nzd', ]) }} as _airbyte_unique_key, - id, - currency, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, date as _airbyte_start_at, lag(date) over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) @@ -112,14 +112,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 3a93bd7be1962..363a39ec25fe7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -58,18 +58,18 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'nzd', + 'id', + 'currency', + 'nzd', ]) }} as _airbyte_unique_key, - id, - currency, - new_column, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - nzd, - usd, + id, + currency, + new_column, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + nzd, + usd, date as _airbyte_start_at, lag(date) over ( partition by cast(id as {{ dbt_utils.type_string() }}), currency, cast(nzd as {{ dbt_utils.type_string() }}) @@ -112,14 +112,14 @@ dedup_data as ( select _airbyte_unique_key, _airbyte_unique_key_scd, - id, - currency, - new_column, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - nzd, - usd, + id, + currency, + new_column, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + nzd, + usd, _airbyte_start_at, _airbyte_end_at, _airbyte_active_row, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql index e360f41b2c90a..73631957ce269 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -20,9 +20,9 @@ scd_data as ( ), '') as varchar )) as _AIRBYTE_UNIQUE_KEY, - ID, - DATE, - PARTITION, + ID, + DATE, + PARTITION, DATE as _AIRBYTE_START_AT, lag(DATE) over ( partition by ID @@ -69,9 +69,9 @@ dedup_data as ( select _AIRBYTE_UNIQUE_KEY, _AIRBYTE_UNIQUE_KEY_SCD, - ID, - DATE, - PARTITION, + ID, + DATE, + PARTITION, _AIRBYTE_START_AT, _AIRBYTE_END_AT, _AIRBYTE_ACTIVE_ROW, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql index 738a31e2d2969..167cdb066cb4c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -56,11 +56,11 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'ID', + 'ID', ]) }} as _AIRBYTE_UNIQUE_KEY, - ID, - DATE, - PARTITION, + ID, + DATE, + PARTITION, DATE as _AIRBYTE_START_AT, lag(DATE) over ( partition by ID @@ -103,9 +103,9 @@ dedup_data as ( select _AIRBYTE_UNIQUE_KEY, _AIRBYTE_UNIQUE_KEY_SCD, - ID, - DATE, - PARTITION, + ID, + DATE, + PARTITION, _AIRBYTE_START_AT, _AIRBYTE_END_AT, _AIRBYTE_ACTIVE_ROW, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql index 098f6d3e91026..20ae8e46add52 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -24,14 +24,14 @@ scd_data as ( ), '') as varchar )) as _AIRBYTE_UNIQUE_KEY, - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - "HKD@spéçiäl & characters", - HKD_SPECIAL___CHARACTERS, - NZD, - USD, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, DATE as _AIRBYTE_START_AT, lag(DATE) over ( partition by ID, CURRENCY, cast(NZD as @@ -82,14 +82,14 @@ dedup_data as ( select _AIRBYTE_UNIQUE_KEY, _AIRBYTE_UNIQUE_KEY_SCD, - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - "HKD@spéçiäl & characters", - HKD_SPECIAL___CHARACTERS, - NZD, - USD, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, _AIRBYTE_START_AT, _AIRBYTE_END_AT, _AIRBYTE_ACTIVE_ROW, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql index 241db9acb9984..2b62f6776a223 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -58,18 +58,18 @@ scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select {{ dbt_utils.surrogate_key([ - 'ID', - 'CURRENCY', - 'NZD', + 'ID', + 'CURRENCY', + 'NZD', ]) }} as _AIRBYTE_UNIQUE_KEY, - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - HKD_SPECIAL___CHARACTERS, - NZD, - USD, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + HKD_SPECIAL___CHARACTERS, + NZD, + USD, DATE as _AIRBYTE_START_AT, lag(DATE) over ( partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) @@ -112,14 +112,14 @@ dedup_data as ( select _AIRBYTE_UNIQUE_KEY, _AIRBYTE_UNIQUE_KEY_SCD, - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - HKD_SPECIAL___CHARACTERS, - NZD, - USD, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + HKD_SPECIAL___CHARACTERS, + NZD, + USD, _AIRBYTE_START_AT, _AIRBYTE_END_AT, _AIRBYTE_ACTIVE_ROW, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json index 9a13299d088e4..9b44f5e68d18a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json @@ -96,6 +96,9 @@ "properties": { "id": { "type": "integer" + }, + "_ab_cdc_updated_at": { + "type": ["null", "number"] } } }, diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 721a38e646682..0768c00364bf2 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -376,12 +376,12 @@ def generate_json_parsing_model(self, from_table: str, column_names: Dict[str, T -- depends_on: {{ from_table }} {{ unnesting_before_query }} select - {%- if parent_hash_id %} +{%- if parent_hash_id %} {{ parent_hash_id }}, - {%- endif %} - {%- for field in fields %} +{%- endif %} +{%- for field in fields %} {{ field }}, - {%- endfor %} +{%- endfor %} {{ col_ab_id }}, {{ col_emitted_at }}, {{ '{{ current_timestamp() }}' }} as {{ col_normalized_at }} @@ -454,12 +454,12 @@ def generate_column_typing_model(self, from_table: str, column_names: Dict[str, -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -- depends_on: {{ from_table }} select - {%- if parent_hash_id %} +{%- if parent_hash_id %} {{ parent_hash_id }}, - {%- endif %} - {%- for field in fields %} +{%- endif %} +{%- for field in fields %} {{ field }}, - {%- endfor %} +{%- endfor %} {{ col_ab_id }}, {{ col_emitted_at }}, {{ '{{ current_timestamp() }}' }} as {{ col_normalized_at }} @@ -573,9 +573,9 @@ def generate_snowflake_timestamp_statement(column_name: str) -> str: template = Template( """ case - {% for format_item in formats %} +{% for format_item in formats %} when {{column_name}} regexp '{{format_item['regex']}}' then to_timestamp_tz({{column_name}}, '{{format_item['format']}}') - {% endfor %} +{% endfor %} when {{column_name}} = '' then NULL else to_timestamp_tz({{column_name}}) end as {{column_name}} @@ -591,12 +591,12 @@ def generate_id_hashing_model(self, from_table: str, column_names: Dict[str, Tup -- depends_on: {{ from_table }} select {{ '{{' }} dbt_utils.surrogate_key([ - {%- if parent_hash_id %} +{%- if parent_hash_id %} {{ parent_hash_id }}, - {%- endif %} - {%- for field in fields %} +{%- endif %} +{%- for field in fields %} {{ field }}, - {%- endfor %} +{%- endfor %} ]) {{ '}}' }} as {{ hash_id }}, tmp.* from {{ from_table }} tmp @@ -649,11 +649,12 @@ def safe_cast_to_string(definition: Dict, column_name: str, destination_type: De return col def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tuple[str, str]]) -> str: - order_null = "is null asc" + cursor_field = self.get_cursor_field(column_names) + order_null = f"is null asc,\n {cursor_field} desc" if self.destination_type.value == DestinationType.ORACLE.value: order_null = "desc nulls last" if self.destination_type.value == DestinationType.MSSQL.value: - # SQL Server treats NULL values as the lowest values, then sorted in ascending order, NULLs come first. + # SQL Server treats NULL values as the lowest values, thus NULLs come last when desc. order_null = "desc" lag_begin = "lag" @@ -687,7 +688,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup quoted_col_cdc_deleted_at = self.name_transformer.normalize_column_name("_ab_cdc_deleted_at", in_jinja=True) quoted_col_cdc_updated_at = self.name_transformer.normalize_column_name("_ab_cdc_updated_at", in_jinja=True) cdc_active_row_pattern = f" and {col_cdc_deleted_at} is null" - cdc_updated_order_pattern = f", {col_cdc_updated_at} desc" + cdc_updated_order_pattern = f"\n {col_cdc_updated_at} desc," cdc_cols = ( f", {cast_begin}{col_cdc_deleted_at}{cast_as}" + "{{ dbt_utils.type_string() }}" @@ -701,14 +702,13 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup if "_ab_cdc_log_pos" in column_names.keys(): col_cdc_log_pos = self.name_transformer.normalize_column_name("_ab_cdc_log_pos") quoted_col_cdc_log_pos = self.name_transformer.normalize_column_name("_ab_cdc_log_pos", in_jinja=True) - cdc_updated_order_pattern += f", {col_cdc_log_pos} desc" + cdc_updated_order_pattern += f"\n {col_cdc_log_pos} desc," cdc_cols += f", {cast_begin}{col_cdc_log_pos}{cast_as}" + "{{ dbt_utils.type_string() }}" + f"{cast_end}" quoted_cdc_cols += f", {quoted_col_cdc_log_pos}" if ( self.destination_type == DestinationType.BIGQUERY - and len(self.cursor_field) == 1 - and is_number(self.properties[self.cursor_field[0]]["type"]) + and is_number(self.properties[self.get_cursor_field_property_name(column_names)]["type"]) ): # partition by float columns is not allowed in BigQuery, cast it to string airbyte_start_at_string = ( @@ -734,7 +734,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup "col_ab_id": self.get_ab_id(), "col_emitted_at": self.get_emitted_at(), "col_normalized_at": self.get_normalized_at(), - "cursor_field": self.get_cursor_field(column_names), + "cursor_field": cursor_field, "enable_left_join_null": enable_left_join_null, "fields": self.list_fields(column_names), "from_table": from_table, @@ -762,9 +762,8 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup row_number() over ( partition by {{ primary_key_partition | join(", ") }} order by - {{ cursor_field }} {{ order_null }}, - {{ cursor_field }} desc, - {{ col_emitted_at }} desc{{ cdc_updated_at_order }} + {{ cursor_field }} {{ order_null }},{{ cdc_updated_at_order }} + {{ col_emitted_at }} desc ) as _airbyte_active_row_num from input_data ),""" @@ -776,11 +775,9 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup {{ lag_begin }}({{ cursor_field }}) over ( partition by {{ primary_key_partition | join(", ") }} order by - {{ cursor_field }} {{ order_null }}, - {{ cursor_field }} desc, - {{ col_emitted_at }} desc{{ cdc_updated_at_order }} - {{ lag_end }} - ) as {{ airbyte_end_at }}""" + {{ cursor_field }} {{ order_null }},{{ cdc_updated_at_order }} + {{ col_emitted_at }} desc + {{ lag_end }}) as {{ airbyte_end_at }}""" ).render(jinja_variables) jinja_variables["scd_columns_sql"] = scd_columns_sql else: @@ -789,16 +786,14 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup lag({{ cursor_field }}) over ( partition by {{ primary_key_partition | join(", ") }} order by - {{ cursor_field }} {{ order_null }}, - {{ cursor_field }} desc, - {{ col_emitted_at }} desc{{ cdc_updated_at_order }} + {{ cursor_field }} {{ order_null }},{{ cdc_updated_at_order }} + {{ col_emitted_at }} desc ) as {{ airbyte_end_at }}, case when row_number() over ( partition by {{ primary_key_partition | join(", ") }} order by - {{ cursor_field }} {{ order_null }}, - {{ cursor_field }} desc, - {{ col_emitted_at }} desc{{ cdc_updated_at_order }} + {{ cursor_field }} {{ order_null }},{{ cdc_updated_at_order }} + {{ col_emitted_at }} desc ) = 1{{ cdc_active_row }} then 1 else 0 end as {{ active_row }}""" ).render(jinja_variables) jinja_variables["scd_columns_sql"] = scd_columns_sql @@ -820,9 +815,9 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup -- build a subset of {{ unique_key }} from rows that are new select distinct {{ '{{' }} dbt_utils.surrogate_key([ - {%- for primary_key in primary_keys %} +{%- for primary_key in primary_keys %} {{ primary_key }}, - {%- endfor %} +{%- endfor %} ]) {{ '}}' }} as {{ unique_key }} from new_data ), @@ -857,17 +852,17 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup scd_data as ( -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key select - {%- if parent_hash_id %} - {{ parent_hash_id }}, - {%- endif %} +{%- if parent_hash_id %} + {{ parent_hash_id }}, +{%- endif %} {{ '{{' }} dbt_utils.surrogate_key([ - {%- for primary_key in primary_keys %} - {{ primary_key }}, - {%- endfor %} +{%- for primary_key in primary_keys %} + {{ primary_key }}, +{%- endfor %} ]) {{ '}}' }} as {{ unique_key }}, - {%- for field in fields %} - {{ field }}, - {%- endfor %} +{%- for field in fields %} + {{ field }}, +{%- endfor %} {{ cursor_field }} as {{ airbyte_start_at }}, {{ scd_columns_sql }}, {{ col_ab_id }}, @@ -895,14 +890,14 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup from scd_data ) select - {%- if parent_hash_id %} - {{ parent_hash_id }}, - {%- endif %} +{%- if parent_hash_id %} + {{ parent_hash_id }}, +{%- endif %} {{ unique_key }}, {{ airbyte_unique_key_scd }}, - {%- for field in fields %} - {{ field }}, - {%- endfor %} +{%- for field in fields %} + {{ field }}, +{%- endfor %} {{ airbyte_start_at }}, {{ airbyte_end_at }}, {{ active_row }}, @@ -915,9 +910,22 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup ).render(jinja_variables) return sql + def get_cursor_field_property_name(self, column_names: Dict[str, Tuple[str, str]]) -> str: + if not self.cursor_field: + if "_ab_cdc_updated_at" in column_names.keys(): + return "_ab_cdc_updated_at" + elif "_ab_cdc_log_pos" in column_names.keys(): + return "_ab_cdc_log_pos" + else: + return self.airbyte_emitted_at + elif len(self.cursor_field) == 1: + return self.cursor_field[0] + else: + raise ValueError(f"Unsupported nested cursor field {'.'.join(self.cursor_field)} for stream {self.stream_name}") + def get_cursor_field(self, column_names: Dict[str, Tuple[str, str]], in_jinja: bool = False) -> str: if not self.cursor_field: - cursor = self.name_transformer.normalize_column_name(self.airbyte_emitted_at, in_jinja) + cursor = self.name_transformer.normalize_column_name(self.get_cursor_field_property_name(column_names), in_jinja) elif len(self.cursor_field) == 1: if not is_airbyte_column(self.cursor_field[0]): cursor = column_names[self.cursor_field[0]][0] @@ -926,7 +934,6 @@ def get_cursor_field(self, column_names: Dict[str, Tuple[str, str]], in_jinja: b cursor = self.cursor_field[0] else: raise ValueError(f"Unsupported nested cursor field {'.'.join(self.cursor_field)} for stream {self.stream_name}") - return cursor def list_primary_keys(self, column_names: Dict[str, Tuple[str, str]]) -> List[str]: @@ -972,15 +979,15 @@ def generate_final_model(self, from_table: str, column_names: Dict[str, Tuple[st -- Final base SQL model -- depends_on: {{ from_table }} select - {%- if parent_hash_id %} +{%- if parent_hash_id %} {{ parent_hash_id }}, - {%- endif %} - {%- if unique_key %} +{%- endif %} +{%- if unique_key %} {{ unique_key }}, - {%- endif %} - {%- for field in fields %} +{%- endif %} +{%- for field in fields %} {{ field }}, - {%- endfor %} +{%- endfor %} {{ col_ab_id }}, {{ col_emitted_at }}, {{ '{{ current_timestamp() }}' }} as {{ col_normalized_at }}, From 4aed0656427533969f1368a9747d209e1ebe5b9f Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Wed, 5 Jan 2022 15:07:12 +0100 Subject: [PATCH 15/16] Indent clickhouse files --- .../scd/test_normalization/dedup_cdc_excluded_scd.sql | 2 +- .../scd/test_normalization/dedup_exchange_rate_scd.sql | 2 +- .../scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql | 2 +- .../scd/test_normalization/dedup_cdc_excluded_scd.sql | 2 +- .../scd/test_normalization/dedup_exchange_rate_scd.sql | 2 +- .../scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql | 2 +- .../normalization/transform_catalog/stream_processor.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index c0ffb324a53dc..0c7c151236fd2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -55,7 +55,7 @@ scd_data as ( _ab_cdc_lsn desc, _ab_cdc_updated_at desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_cdc_excluded_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index f30017d3cd5e5..c1e8e6cb63fec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -62,7 +62,7 @@ scd_data as ( date is null asc, date desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 04d74982d7b2c..eedc913fd45a5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -50,7 +50,7 @@ scd_data as ( _ab_cdc_updated_at is null asc, _ab_cdc_updated_at desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_renamed_dedup_cdc_excluded_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index c9b7736da2845..a20276296c922 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -83,7 +83,7 @@ scd_data as ( _ab_cdc_lsn desc, _ab_cdc_updated_at desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_cdc_excluded_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index e5e0e0f749ffe..1d94573fc99e1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -88,7 +88,7 @@ scd_data as ( date is null asc, date desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index c79c9d716682e..9e3c81ac18178 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -78,7 +78,7 @@ scd_data as ( _ab_cdc_updated_at is null asc, _ab_cdc_updated_at desc, _airbyte_emitted_at desc - ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, + ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) as _airbyte_end_at, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_renamed_dedup_cdc_excluded_hashid diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 0768c00364bf2..aeb403351a6df 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -664,7 +664,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup # ClickHouse doesn't support lag() yet, this is a workaround solution # Ref: https://clickhouse.com/docs/en/sql-reference/window-functions/ lag_begin = "anyOrNull" - lag_end = "ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING" + lag_end = " ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING" input_data_table = "input_data_with_active_row_num" enable_left_join_null = "" From 0771ffead97165a5e7ebc50449ec393c3c1aaea4 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Thu, 6 Jan 2022 15:02:31 +0100 Subject: [PATCH 16/16] Fix when no cursor --- .../normalization/transform_catalog/stream_processor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index aeb403351a6df..8cae3703bc9fc 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -708,6 +708,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup if ( self.destination_type == DestinationType.BIGQUERY + and self.get_cursor_field_property_name(column_names) != self.airbyte_emitted_at and is_number(self.properties[self.get_cursor_field_property_name(column_names)]["type"]) ): # partition by float columns is not allowed in BigQuery, cast it to string