Feat/clickhouse support

CONTRIBUTING.md

@@ -78,6 +78,7 @@ Tox will take care of installing the dependencies for each environment, so you d
     tox -e integration_snowflake # For the Snowflake tests
     tox -e integration_databricks # For the Databricks tests
     tox -e integration_bigquery # For the BigQuery tests
+    tox -e integration_clickhouse # For the Clickhouse tests
     ```
 
 The Spark tests require installing the [ODBC driver](https://www.databricks.com/spark/odbc-drivers-download). On a Mac,

README.md

@@ -20,6 +20,7 @@ The package currently supports
 - Google BigQuery :white_check_mark:
 - Postgres :white_check_mark:
 - SQL Server :white_check_mark:
+- Clickhouse :white_check_mark:
 
 Models included:
 

integration_test_project/example-env.sh

@@ -16,6 +16,9 @@ export DBT_ENV_SECRET_DATABRICKS_TOKEN=
 export DBT_ENV_SECRET_GCP_PROJECT=
 export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dylib on a Mac
 export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path
+export DBT_ENV_SECRET_CLICKHOUSE_HOST=
+export DBT_ENV_SECRET_CLICKHOUSE_USER=
+export DBT_ENV_SECRET_CLICKHOUSE_PASSWORD=
 
 # dbt environment variables, change these
 export DBT_VERSION="1_5_0"

integration_test_project/models/sources.yml

@@ -2,7 +2,7 @@ version: 2
 
 sources:
   - name: dummy_source
-    database: "{% if target.type not in ('spark', 'databricks') %}{{ var('dbt_artifacts_database', target.database) }}{% endif %}"
+    database: "{% if target.type not in ('spark', 'databricks', 'clickhouse') %}{{ var('dbt_artifacts_database', target.database) }}{% endif %}"
     schema: "{{ target.schema }}"
     freshness:
       error_after: {count: 24, period: hour}

integration_test_project/profiles.yml

@@ -64,3 +64,11 @@ dbt_artifacts:
       Encrypt: False
       user: dbt
       password: "123"
+    clickhouse:
+      type: clickhouse
+      host: "{{ env_var('DBT_ENV_SECRET_CLICKHOUSE_HOST') }}"
+      port: 8123
+      user: "{{ env_var('DBT_ENV_SECRET_CLICKHOUSE_USER') }}"
+      password: "{{ env_var('DBT_ENV_SECRET_CLICKHOUSE_PASSWORD') }}"
+      schema: dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }}
+      threads: 8

macros/database_specific_helpers/type_helpers.sql

@@ -8,6 +8,10 @@
    {{ return(api.Column.translate_type("boolean")) }}
 {% endmacro %}
 
+{% macro clickhouse__type_boolean() %}
+   Nullable(Boolean)
+{% endmacro %}
+
 {#- JSON -#}
 
 {% macro type_json() %}
@@ -26,6 +30,10 @@
    json
 {% endmacro %}
 
+{% macro clickhouse__type_json() %}
+   Nullable(String)
+{% endmacro %}
+
 {#- ARRAY -#}
 
 {% macro type_array() %}
@@ -43,3 +51,64 @@
 {% macro bigquery__type_array() %}
    array<string>
 {% endmacro %}
+
+{% macro clickhouse__type_array() %}
+   Array(String)
+{% endmacro %}
+
+{#- STRING -#}
+
+{% macro type_string() %}
+    {{ return(adapter.dispatch('type_string', 'dbt_artifacts')()) }}
+{% endmacro %}
+
+{% macro default__type_string() %}
+  {{ return(api.Column.translate_type("string")) }}
+{% endmacro %}
+
+{% macro clickhouse__type_string() %}
+  Nullable(String)
+{% endmacro %}
+
+{#- TIMESTAMP -#}
+
+{% macro type_timestamp() %}
+    {{ return(adapter.dispatch('type_timestamp', 'dbt_artifacts')()) }}
+{% endmacro %}
+
+{% macro default__type_timestamp() %}
+  {{ return(api.Column.translate_type("timestamp")) }}
+{% endmacro %}
+
+{% macro clickhouse__type_timestamp() %}
+  Nullable(DateTime)
+{% endmacro %}
+
+{#- INT -#}
+
+{% macro type_int() %}
+    {{ return(adapter.dispatch('type_int', 'dbt_artifacts')()) }}
+{% endmacro %}
+
+{% macro default__type_int() %}
+  {{ return(api.Column.translate_type("integer")) }}
+{% endmacro %}
+
+{% macro clickhouse__type_int() %}
+  Nullable(Int32)
+{% endmacro %}
+
+{#- FLOAT -#}
+
+{% macro type_float() %}
+    {{ return(adapter.dispatch('type_float', 'dbt_artifacts')()) }}
+{% endmacro %}
+
+{% macro default__type_float() %}
+  {{ return(api.Column.translate_type("float")) }}
+{% endmacro %}
+
+{% macro clickhouse__type_float() %}
+  Nullable(Float32)
+{% endmacro %}
+

macros/upload_individual_datasets/upload_exposures.sql

@@ -154,3 +154,7 @@
     {% endif %}
 {% endmacro -%}
 
+{% macro clickhouse__get_exposures_dml_sql(exposures) -%}
+{{ return(dbt_artifacts.postgres__get_exposures_dml_sql(exposures)) }}
+{%- endmacro %}
+

macros/upload_individual_datasets/upload_invocations.sql

@@ -296,3 +296,6 @@
 
 {% endmacro -%}
 
+{% macro clickhouse__get_invocations_dml_sql() -%}
+{{ return(dbt_artifacts.postgres__get_invocations_dml_sql()) }}
+{% endmacro -%}

macros/upload_individual_datasets/upload_model_executions.sql

@@ -246,3 +246,6 @@
     {% endif %}
 {% endmacro -%}
 
+{% macro clickhouse__get_model_executions_dml_sql(models) -%}
+{{ return(dbt_artifacts.postgres__get_model_executions_dml_sql(models)) }}
+{% endmacro -%}

macros/upload_individual_datasets/upload_models.sql

@@ -168,3 +168,6 @@
     {% endif %}
 {% endmacro -%}
 
+{% macro clickhouse__get_models_dml_sql(models) -%}
+{{ return(dbt_artifacts.postgres__get_models_dml_sql(models)) }}
+{%- endmacro %}

macros/upload_individual_datasets/upload_seed_executions.sql

@@ -260,3 +260,6 @@
     {% endif %}
 {% endmacro -%}
 
+{% macro clickhouse__get_seed_executions_dml_sql(seeds) -%}
+{{ return(dbt_artifacts.postgres__get_seed_executions_dml_sql(seeds)) }}
+{%- endmacro %}

macros/upload_individual_datasets/upload_seeds.sql

@@ -143,3 +143,6 @@
     {% endif %}
 {% endmacro -%}
 
+{% macro clickhouse__get_seeds_dml_sql(seeds) -%}
+{{ return(dbt_artifacts.postgres__get_seeds_dml_sql(seeds)) }}
+{%- endmacro %}

macros/upload_individual_datasets/upload_snapshot_executions.sql

@@ -260,3 +260,6 @@
     {% endif %}
 {% endmacro -%}
 
+{% macro clickhouse__get_snapshot_executions_dml_sql(snapshots) -%}
+{{ return(dbt_artifacts.postgres__get_snapshot_executions_dml_sql(snapshots)) }}
+{%- endmacro %}

macros/upload_individual_datasets/upload_snapshots.sql

@@ -157,3 +157,6 @@
     {% endif %}
 {% endmacro -%}
 
+{% macro clickhouse__get_snapshots_dml_sql(snapshots) -%}
+{{ return(dbt_artifacts.postgres__get_snapshots_dml_sql(snapshots)) }}
+{%- endmacro %}

macros/upload_individual_datasets/upload_sources.sql

@@ -107,6 +107,9 @@
     {% endif %}
 {%- endmacro %}
 
+{% macro clickhouse__get_sources_dml_sql(sources) -%}
+{{ return(dbt_artifacts.postgres__get_sources_dml_sql(sources)) }}
+{%- endmacro %}
 
 {% macro sqlserver__get_sources_dml_sql(sources) -%}
 

macros/upload_individual_datasets/upload_test_executions.sql

@@ -241,4 +241,6 @@
     {% endif %}
 {% endmacro -%}
 
-
+{% macro clickhouse__get_test_executions_dml_sql(tests) -%}
+{{ return(dbt_artifacts.postgres__get_test_executions_dml_sql(tests)) }}
+{%- endmacro %}

macros/upload_individual_datasets/upload_tests.sql

@@ -95,6 +95,9 @@
     {% endif %}
 {%- endmacro %}
 
+{% macro clickhouse__get_tests_dml_sql(tests) -%}
+{{ return(dbt_artifacts.postgres__get_tests_dml_sql(tests)) }}
+{%- endmacro %}
 
 {% macro sqlserver__get_tests_dml_sql(tests) -%}
 

macros/upload_results/insert_into_metadata_table.sql

@@ -70,6 +70,18 @@
 
 {%- endmacro %}
 
+{% macro clickhouse__insert_into_metadata_table(relation, fields, content) -%}
+
+    {% set insert_into_table_query %}
+    insert into {{ relation }} {{ fields }}
+    values
+    {{ content }}
+    {% endset %}
+
+    {% do run_query(insert_into_table_query) %}
+
+{%- endmacro %}
+
 {% macro default__insert_into_metadata_table(relation, fields, content) -%}
 {%- endmacro %}
 

models/sources/exposures.sql

@@ -18,8 +18,16 @@ select
     , cast(null as {{ type_string() }}) as description
     , cast(null as {{ type_string() }}) as url
     , cast(null as {{ type_string() }}) as package_name
-    , cast(null as {{ type_array() }}) as depends_on_nodes
-    , cast(null as {{ type_array() }}) as tags
+    {% if target.type == "clickhouse" %}
+        , cast(null as {{ type_string() }}) as depends_on_nodes
+    {% else %}
+        , cast(null as {{ type_array() }}) as depends_on_nodes
+    {% endif %}
+    {% if target.type == "clickhouse" %}
+        , cast(null as {{ type_string() }}) as tags
+    {% else %}
+        , cast(null as {{ type_array() }}) as tags
+    {% endif %}
     , cast(null as {{ type_json() }}) as all_results
 from dummy_cte
 where 1 = 0

models/sources/models.sql

@@ -12,12 +12,20 @@ select
     {% else %} schema
     {% endif %},
     cast(null as {{ type_string() }}) as name,
-    cast(null as {{ type_array() }}) as depends_on_nodes,
+    {% if target.type == "clickhouse" %}
+        cast(null as {{ type_string() }}) as depends_on_nodes,
+    {% else %}
+        cast(null as {{ type_array() }}) as depends_on_nodes,
+    {% endif %}
     cast(null as {{ type_string() }}) as package_name,
     cast(null as {{ type_string() }}) as path,
     cast(null as {{ type_string() }}) as checksum,
     cast(null as {{ type_string() }}) as materialization,
-    cast(null as {{ type_array() }}) as tags,
+    {% if target.type == "clickhouse" %}
+        cast(null as {{ type_string() }}) as tags,
+    {% else %}
+        cast(null as {{ type_array() }}) as tags,
+    {% endif %}
     cast(null as {{ type_json() }}) as meta,
     cast(null as {{ type_string() }}) as alias,
     cast(null as {{ type_json() }}) as all_results

models/sources/snapshots.sql

@@ -12,7 +12,11 @@ select
     {% else %} schema
     {% endif %},
     cast(null as {{ type_string() }}) as name,
-    cast(null as {{ type_array() }}) as depends_on_nodes,
+    {% if target.type == "clickhouse" %}
+        cast(null as {{ type_string() }}) as depends_on_nodes,
+    {% else %}
+        cast(null as {{ type_array() }}) as depends_on_nodes,
+    {% endif %}
     cast(null as {{ type_string() }}) as package_name,
     cast(null as {{ type_string() }}) as path,
     cast(null as {{ type_string() }}) as checksum,

models/sources/tests.sql

@@ -11,10 +11,19 @@ select
     , cast(null as {{ type_string() }}) as node_id
     , cast(null as {{ type_timestamp() }}) as run_started_at
     , cast(null as {{ type_string() }}) as name
-    , cast(null as {{ type_array() }}) as depends_on_nodes
+    {% if target.type == "clickhouse" %}
+        , cast(null as {{ type_string() }}) as depends_on_nodes
+    {% else %}
+        , cast(null as {{ type_array() }}) as depends_on_nodes
+    {% endif %}
     , cast(null as {{ type_string() }}) as package_name
     , cast(null as {{ type_string() }}) as test_path
-    , cast(null as {{ type_array() }}) as tags
+    {% if target.type == "clickhouse" %}
+        , cast(null as {{ type_string() }}) as tags
+    {% else %}
+        , cast(null as {{ type_array() }}) as tags
+    {% endif %}
     , cast(null as {{ type_json() }}) as all_results
 from dummy_cte
 where 1 = 0
+

models/staging/stg_dbt__model_executions.sql

@@ -9,7 +9,9 @@ with
             node_id,
             run_started_at,
             was_full_refresh,
-            {{ split_part("thread_id", "'-'", 2) }} as thread_id,
+            {% if target.type == "clickhouse" %} {{ split_part("coalesce(thread_id, '')", "'-'", 2) }} as thread_id,
+            {% else %} {{ split_part("thread_id", "'-'", 2) }} as thread_id,
+            {% endif %}
             status,
             compile_started_at,
             query_completed_at,

models/staging/stg_dbt__seed_executions.sql

@@ -9,7 +9,9 @@ with
             node_id,
             run_started_at,
             was_full_refresh,
-            {{ split_part("thread_id", "'-'", 2) }} as thread_id,
+            {% if target.type == "clickhouse" %} {{ split_part("coalesce(thread_id, '')", "'-'", 2) }} as thread_id,
+            {% else %} {{ split_part("thread_id", "'-'", 2) }} as thread_id,
+            {% endif %}
             status,
             compile_started_at,
             query_completed_at,

models/staging/stg_dbt__snapshot_executions.sql

@@ -9,7 +9,9 @@ with
             node_id,
             run_started_at,
             was_full_refresh,
-            {{ split_part("thread_id", "'-'", 2) }} as thread_id,
+            {% if target.type == "clickhouse" %} {{ split_part("coalesce(thread_id, '')", "'-'", 2) }} as thread_id,
+            {% else %} {{ split_part("thread_id", "'-'", 2) }} as thread_id,
+            {% endif %}
             status,
             compile_started_at,
             query_completed_at,

models/staging/stg_dbt__test_executions.sql

@@ -14,7 +14,10 @@ with
             , node_id
             , run_started_at
             , was_full_refresh
-            , {{ split_part('thread_id', "'-'", 2) }} as thread_id
+            ,
+            {% if target.type == "clickhouse" %} {{ split_part("coalesce(thread_id, '')", "'-'", 2) }} as thread_id
+            {% else %} {{ split_part("thread_id", "'-'", 2) }} as thread_id
+            {% endif %}
             , status
             , compile_started_at
             , query_completed_at

tox.ini

@@ -84,6 +84,9 @@ passenv =
     DBT_ENV_SECRET_GCP_PROJECT
     DBT_ENV_SPARK_DRIVER_PATH
     DBT_ENV_SPARK_ENDPOINT
+    DBT_ENV_SECRET_CLICKHOUSE_HOST
+    DBT_ENV_SECRET_CLICKHOUSE_USER
+    DBT_ENV_SECRET_CLICKHOUSE_PASSWORD
     GOOGLE_APPLICATION_CREDENTIALS
     DBT_CLOUD_PROJECT_ID
     DBT_CLOUD_JOB_ID
@@ -430,3 +433,11 @@ commands =
     dbt clean
     dbt deps
     dbt build --target sqlserver
+
+[testenv:integration_clickhouse]
+changedir = integration_test_project
+deps = dbt-clickhouse~=1.8.0
+commands =
+    dbt clean
+    dbt deps
+    dbt build --target clickhouse