diff --git a/.circleci/config.yml b/.circleci/config.yml index 8869c8bb..2432273b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ jobs: integration-postgres: docker: - - image: circleci/python:3.6.13-stretch + - image: cimg/python:3.9.9 - image: circleci/postgres:9.6.5-alpine-ram steps: @@ -24,7 +24,7 @@ jobs: integration-redshift: docker: - - image: circleci/python:3.6.3-stretch + - image: cimg/python:3.9.9 steps: - checkout - run: @@ -35,7 +35,7 @@ jobs: integration-snowflake: docker: - - image: circleci/python:3.6.3-stretch + - image: cimg/python:3.9.9 steps: - checkout - run: @@ -48,7 +48,7 @@ jobs: environment: BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" docker: - - image: circleci/python:3.6.3-stretch + - image: cimg/python:3.9.9 steps: - checkout - run: diff --git a/CHANGELOG.md b/CHANGELOG.md index df6704cc..6ad5e3e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,24 @@ # dbt-utils v0.8.0 ## 🚨 Breaking changes +- dbt ONE POINT OH is here! This version of dbt-utils requires _any_ version (minor and patch) of v1, which means far less need for compatibility releases in the future. - The partition column in the `mutually_exclusive_ranges` test is now always called `partition_by_col`. This enables compatibility with `--store-failures` when multiple columns are concatenated together. If you have models built on top of the failures table, update them to reflect the new column name. ([#423](https://github.com/dbt-labs/dbt-utils/issues/423), [#430](https://github.com/dbt-labs/dbt-utils/pull/430)) ## Contributors: - [codigo-ergo-sum](https://github.com/codigo-ergo-sum) (#430) -# dbt-utils v0.7.4 +# dbt-utils 0.7.5 🚨 This is a compatibility release in preparation for `dbt-core` v1.0.0 (🎉). Projects using dbt-utils 0.7.4 with dbt-core v1.0.0 can expect to see a deprecation warning. This will be resolved in dbt_utils v0.8.0. +## Fixes +- Regression in `get_column_values()` where the default would not be respected if the model didn't exist. ([#444](https://github.com/dbt-labs/dbt-utils/issues/444), [#448](https://github.com/dbt-labs/dbt-utils/pull/448)) + +## Under the hood +- get_url_host() macro now correctly handles URLs beginning with android-app:// (#426) + +## Contributors: +- [foundinblank](https://github.com/foundinblank) + +# dbt-utils v0.7.4 ## Fixes - `get_column_values()` now works correctly with mixed-quoting styles on Snowflake ([#424](https://github.com/dbt-labs/dbt-utils/issues/424), [#440](https://github.com/dbt-labs/dbt-utils/pull/440)) - Remove extra semicolon in `insert_by_period` materialization that was causing errors ([#439](https://github.com/dbt-labs/dbt-utils/pull/439)) diff --git a/dbt_project.yml b/dbt_project.yml index aaaf9d49..833501d8 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,11 +1,11 @@ name: 'dbt_utils' -version: '0.7.0' +version: '0.1.0' -require-dbt-version: [">=0.20.0", "<=1.0.0"] +require-dbt-version: [">=1.0.0", "<2.0.0"] config-version: 2 target-path: "target" -clean-targets: ["target", "dbt_modules"] +clean-targets: ["target", "dbt_modules", "dbt_packages"] macro-paths: ["macros"] log-path: "logs" diff --git a/integration_tests/data/sql/data_get_column_values_dropped.csv b/integration_tests/data/sql/data_get_column_values_dropped.csv new file mode 100644 index 00000000..5651149b --- /dev/null +++ b/integration_tests/data/sql/data_get_column_values_dropped.csv @@ -0,0 +1,12 @@ +field +a +b +c +d +e +f +g +g +g +g +g diff --git a/integration_tests/data/web/data_url_host.csv b/integration_tests/data/web/data_url_host.csv index 36daaa00..2350bb7a 100644 --- a/integration_tests/data/web/data_url_host.csv +++ b/integration_tests/data/web/data_url_host.csv @@ -4,4 +4,4 @@ http://witanddelight.com/2018/01/tips-tricks-how-run-half-marathon-first-time/,w https://www.nytimes.com/2018/01/01/blog,www.nytimes.com android-app://m.facebook.com/,m.facebook.com docs.nytimes.com/2021/01/01/index.js?utm_source=google,docs.nytimes.com -https://m.facebook.com/,m.facebook.com +https://m.facebook.com/,m.facebook.com \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 40db9050..bb6d147b 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -8,16 +8,17 @@ profile: 'integration_tests' config-version: 2 -source-paths: ["models"] +model-paths: ["models"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["data"] macro-paths: ["macros"] target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_modules" + - "dbt_packages" dispatch: - macro_namespace: 'dbt_utils' @@ -54,6 +55,11 @@ seeds: sql: data_events_20180103: +schema: events + + data_get_column_values_dropped: + # this.incorporate() to hardcode the node's type as otherwise dbt doesn't know it yet + +post-hook: "{% do adapter.drop_relation(this.incorporate(type='table')) %}" + schema_tests: data_test_sequential_timestamps: diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql new file mode 100644 index 00000000..7f9fa3a4 --- /dev/null +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -0,0 +1,26 @@ + +{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} + +with expected as ( + select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected_column_value union all + select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected_column_value +), + +actual as ( + + {% for val in column_values %} + select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual_column_value + {% if not loop.last %} + union all + {% endif %} + {% endfor %} +), + +failures as ( + select * from actual + where actual.actual_column_value not in ( + select expected.expected_column_value from expected + ) +) + +select * from failures \ No newline at end of file diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index bc05fb07..57b150a6 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -15,15 +15,18 @@ {# TODO: Change the method signature in a future 0.x.0 release #} {%- set target_relation = table -%} + {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} + {% set relation_exists = (load_relation(target_relation)) is not none %} + {%- call statement('get_column_values', fetch_result=true) %} - {%- if not target_relation and default is none -%} + {%- if not relation_exists and default is none -%} - {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ table ~ " does not exist and no default value was provided.") }} + {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} - {%- elif not target_relation and default is not none -%} + {%- elif not relation_exists and default is not none -%} - {{ log("Relation " ~ table ~ " does not exist. Returning the default value: " ~ default) }} + {{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} {{ return(default) }}