diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d4cf466..58791b7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt-utils v0.6.1 + +## Fixes +- Fix the logic in `get_tables_by_pattern_sql` for matching a schema pattern on BigQuery ([#275](https://github.com/fishtown-analytics/dbt-utils/pull/275/)) + # dbt-utils v0.6.0 ## Breaking changes diff --git a/integration_tests/data/sql/data_events_20180103.csv b/integration_tests/data/sql/data_events_20180103.csv new file mode 100644 index 00000000..4379e0d4 --- /dev/null +++ b/integration_tests/data/sql/data_events_20180103.csv @@ -0,0 +1,3 @@ +user_id,event +5,play +6,pause diff --git a/integration_tests/data/sql/data_union_events_expected.csv b/integration_tests/data/sql/data_union_events_expected.csv new file mode 100644 index 00000000..86303b2c --- /dev/null +++ b/integration_tests/data/sql/data_union_events_expected.csv @@ -0,0 +1,7 @@ +user_id,event +1,play +2,pause +3,play +4,pause +5,play +6,pause diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index f0840fed..c25df991 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -46,3 +46,7 @@ seeds: num_buckets: integer min_value: float max_value: float + + sql: + data_events_20180103: + +schema: events \ No newline at end of file diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index 6c6f891b..dcf0b8ad 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -50,7 +50,7 @@ models: values: - '5' - - name: test_get_tables_by_prefix_and_union + - name: test_get_relations_by_prefix_and_union columns: - name: event tests: @@ -117,3 +117,8 @@ models: tests: - dbt_utils.equality: compare_model: ref('data_union_expected') + + - name: test_get_relations_by_pattern + tests: + - dbt_utils.equality: + compare_model: ref('data_union_events_expected') diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index b10d8eef..52ab07a2 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,4 +1,16 @@ {{ config(materialized = 'table') }} -{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_%') %} -{{ dbt_utils.union_relations(relations) }} \ No newline at end of file +{% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %} + +with unioned as ( + + {{ dbt_utils.union_relations(relations) }} + +) + +select + + user_id, + event + +from unioned diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index df47aa23..426c676a 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -5,7 +5,7 @@ {% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - select distinct + select distinct table_schema as "table_schema", table_name as "table_name" from {{database}}.information_schema.tables where table_schema ilike '{{ schema_pattern }}' @@ -16,13 +16,51 @@ {% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - - select distinct - table_schema, table_name - from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES - where table_schema = '{{schema_pattern}}' - and lower(table_name) like lower ('{{table_pattern}}') - and lower(table_name) not like lower ('{{exclude}}') + {% if '%' in schema_pattern %} + {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %} + {% else %} + {% set schemata=[schema_pattern] %} + {% endif %} + + {% set sql %} + {% for schema in schemata %} + select distinct + table_schema, table_name + + from {{ adapter.quote(database) }}.{{ schema }}.INFORMATION_SCHEMA.TABLES + where lower(table_name) like lower ('{{ table_pattern }}') + and lower(table_name) not like lower ('{{ exclude }}') + + {% if not loop.last %} union all {% endif %} + + {% endfor %} + {% endset %} + + {{ return(sql) }} + +{% endmacro %} + + +{% macro _bigquery__get_matching_schemata(schema_pattern, database) %} + {% if execute %} + + {% set sql %} + select schema_name from {{ adapter.quote(database) }}.INFORMATION_SCHEMA.SCHEMATA + where lower(schema_name) like lower('{{ schema_pattern }}') + {% endset %} + + {% set results=run_query(sql) %} + + {% set schemata=results.columns['schema_name'].values() %} + + {{ return(schemata) }} + + {% else %} + + {{ return([]) }} + + {% endif %} + {% endmacro %}