From bc3930eb224a57c1bd0991b75a26a496c53da8e6 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 31 Mar 2023 14:09:15 +0100 Subject: [PATCH 1/6] Add documentation for sources --- models/sources/exposures.yml | 30 ++++++++++++++++++ models/sources/invocations.yml | 44 ++++++++++++++++++++++++++ models/sources/model_executions.yml | 38 ++++++++++++++++++++++ models/sources/models.yml | 34 ++++++++++++++++++++ models/sources/seed_executions.yml | 34 ++++++++++++++++++++ models/sources/seeds.yml | 26 +++++++++++++++ models/sources/snapshot_executions.yml | 34 ++++++++++++++++++++ models/sources/snapshots.yml | 32 +++++++++++++++++++ models/sources/sources.yml | 30 ++++++++++++++++++ models/sources/test_executions.yml | 32 +++++++++++++++++++ models/sources/tests.yml | 24 ++++++++++++++ 11 files changed, 358 insertions(+) create mode 100644 models/sources/exposures.yml create mode 100644 models/sources/invocations.yml create mode 100644 models/sources/model_executions.yml create mode 100644 models/sources/models.yml create mode 100644 models/sources/seed_executions.yml create mode 100644 models/sources/seeds.yml create mode 100644 models/sources/snapshot_executions.yml create mode 100644 models/sources/snapshots.yml create mode 100644 models/sources/sources.yml create mode 100644 models/sources/test_executions.yml create mode 100644 models/sources/tests.yml diff --git a/models/sources/exposures.yml b/models/sources/exposures.yml new file mode 100644 index 00000000..273a1327 --- /dev/null +++ b/models/sources/exposures.yml @@ -0,0 +1,30 @@ +version: 2 + +models: + - name: exposures + description: Base model for data about exposure exections. One row per node per run. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: depends_on_nodes + description: '{{ doc("depends_on_nodes") }}' + - name: description + description: '{{ doc("description") }}' + - name: maturity + description: '{{ doc("maturity") }}' + - name: name + description: '{{ doc("name") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: owner + description: '{{ doc("owner") }}' + - name: package_name + description: '{{ doc("package_name") }}' + - name: path + description: '{{ doc("path") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: type + description: '{{ doc("type") }}' + - name: url + description: '{{ doc("url") }}' diff --git a/models/sources/invocations.yml b/models/sources/invocations.yml new file mode 100644 index 00000000..82177128 --- /dev/null +++ b/models/sources/invocations.yml @@ -0,0 +1,44 @@ +version: 2 + +models: + - name: invocations + description: Base model for data about the invocations of dbt. One row per run. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: dbt_version + description: '{{ doc("dbt_version") }}' + - name: project_name + description: '{{ doc("project_name") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: dbt_command + description: '{{ doc("dbt_command") }}' + - name: full_refresh_flag + description: '{{ doc("full_refresh_flag") }}' + - name: target_profile_name + description: '{{ doc("target_profile_name") }}' + - name: target_name + description: '{{ doc("target_name") }}' + - name: target_schema + description: '{{ doc("target_schema") }}' + - name: target_threads + description: '{{ doc("target_threads") }}' + - name: dbt_cloud_project_id + description: '{{ doc("dbt_cloud_project_id") }}' + - name: dbt_cloud_job_id + description: '{{ doc("dbt_cloud_job_id") }}' + - name: dbt_cloud_run_id + description: '{{ doc("dbt_cloud_run_id") }}' + - name: dbt_cloud_run_reason_category + description: '{{ doc("dbt_cloud_run_reason_category") }}' + - name: dbt_cloud_run_reason + description: '{{ doc("dbt_cloud_run_reason") }}' + - name: env_vars + description: '{{ doc("env_vars") }}' + - name: dbt_vars + description: '{{ doc("dbt_vars") }}' + - name: invocation_args + description: '{{ doc("invocation_args") }}' + - name: dbt_custom_envs + description: '{{ doc("dbt_custom_envs") }}' diff --git a/models/sources/model_executions.yml b/models/sources/model_executions.yml new file mode 100644 index 00000000..65b7368a --- /dev/null +++ b/models/sources/model_executions.yml @@ -0,0 +1,38 @@ +version: 2 + +models: + - name: model_executions + description: Base model for data about model executions. One row per model execution. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' + - name: materialization + description: '{{ doc("materialization") }}' + - name: name + description: '{{ doc("name") }}' + - name: alias + description: '{{ doc("alias") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: query_completed_at + description: '{{ doc("query_completed_at") }}' + - name: rows_affected + description: '{{ doc("rows_affected") }}' + - name: bytes_processed + description: '{{ doc("bytes_processed") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: status + description: '{{ doc("status") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' + - name: was_full_refresh + description: '{{ doc("was_full_refresh") }}' + - name: message + description: '{{ doc("message") }}' diff --git a/models/sources/models.yml b/models/sources/models.yml new file mode 100644 index 00000000..3e77aa79 --- /dev/null +++ b/models/sources/models.yml @@ -0,0 +1,34 @@ +version: 2 + +models: + - name: models + description: Staging model that contains metadata about model executions. One row per node per run. + columns: + - name: checksum + description: '{{ doc("checksum") }}' + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: database + description: '{{ doc("database") }}' + - name: depends_on_nodes + description: '{{ doc("depends_on_nodes") }}' + - name: materialization + description: '{{ doc("materialization") }}' + - name: name + description: '{{ doc("name") }}' + - name: alias + description: '{{ doc("alias") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: package_name + description: '{{ doc("package_name") }}' + - name: path + description: '{{ doc("path") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: tags + description: '{{ doc("tags") }}' + - name: meta + description: '{{ doc("meta") }}' diff --git a/models/sources/seed_executions.yml b/models/sources/seed_executions.yml new file mode 100644 index 00000000..c6417dc8 --- /dev/null +++ b/models/sources/seed_executions.yml @@ -0,0 +1,34 @@ +version: 2 + +models: + - name: seed_executions + description: Base model for data about seed executions. One row per seed execution. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' + - name: materialization + description: '{{ doc("materialization") }}' + - name: name + description: '{{ doc("name") }}' + - name: alias + description: '{{ doc("alias") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: query_completed_at + description: '{{ doc("query_completed_at") }}' + - name: rows_affected + description: '{{ doc("rows_affected") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: status + description: '{{ doc("status") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' + - name: was_full_refresh + description: '{{ doc("was_full_refresh") }}' + - name: message + description: '{{ doc("message") }}' diff --git a/models/sources/seeds.yml b/models/sources/seeds.yml new file mode 100644 index 00000000..46723eec --- /dev/null +++ b/models/sources/seeds.yml @@ -0,0 +1,26 @@ +version: 2 + +models: + - name: seeds + description: Base model for metadata about seed executions. One row per node per run. + columns: + - name: checksum + description: '{{ doc("checksum") }}' + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: database + description: '{{ doc("database") }}' + - name: name + description: '{{ doc("name") }}' + - name: alias + description: '{{ doc("alias") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: package_name + description: '{{ doc("package_name") }}' + - name: path + description: '{{ doc("path") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: schema + description: '{{ doc("schema") }}' diff --git a/models/sources/snapshot_executions.yml b/models/sources/snapshot_executions.yml new file mode 100644 index 00000000..5a5ce560 --- /dev/null +++ b/models/sources/snapshot_executions.yml @@ -0,0 +1,34 @@ +version: 2 + +models: + - name: snapshot_executions + description: Base model for data about snapshot executions. One row per snapshot execution. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' + - name: materialization + description: '{{ doc("materialization") }}' + - name: name + description: '{{ doc("name") }}' + - name: alias + description: '{{ doc("alias") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: query_completed_at + description: '{{ doc("query_completed_at") }}' + - name: rows_affected + description: '{{ doc("rows_affected") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: status + description: '{{ doc("status") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' + - name: was_full_refresh + description: '{{ doc("was_full_refresh") }}' + - name: message + description: '{{ doc("message") }}' diff --git a/models/sources/snapshots.yml b/models/sources/snapshots.yml new file mode 100644 index 00000000..b78b2f8a --- /dev/null +++ b/models/sources/snapshots.yml @@ -0,0 +1,32 @@ +version: 2 + +models: + - name: snapshots + description: Base model for metadata about seed executions. One row per node per run. + columns: + - name: checksum + description: '{{ doc("checksum") }}' + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: database + description: '{{ doc("database") }}' + - name: depends_on_nodes + description: '{{ doc("depends_on_nodes") }}' + - name: name + description: '{{ doc("name") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: package_name + description: '{{ doc("package_name") }}' + - name: path + description: '{{ doc("path") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: snapshot_execution_id + description: '{{ doc("snapshot_execution_id") }}' + - name: strategy + description: '{{ doc("strategy") }}' + - name: alias + description: '{{ doc("alias") }}' diff --git a/models/sources/sources.yml b/models/sources/sources.yml new file mode 100644 index 00000000..b538ce8f --- /dev/null +++ b/models/sources/sources.yml @@ -0,0 +1,30 @@ +version: 2 + +models: + - name: sources + description: Base model for data about sources. One row per node per execution. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: database + description: '{{ doc("database") }}' + - name: freshness + description: '{{ doc("freshness") }}' + - name: identifier + description: '{{ doc("identifier") }}' + - name: loaded_at_field + description: '{{ doc("loaded_at_field") }}' + - name: loader + description: '{{ doc("loader") }}' + - name: name + description: '{{ doc("name") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: source_execution_id + description: '{{ doc("source_execution_id") }}' + - name: source_name + description: '{{ doc("source_name") }}' diff --git a/models/sources/test_executions.yml b/models/sources/test_executions.yml new file mode 100644 index 00000000..8a76de5c --- /dev/null +++ b/models/sources/test_executions.yml @@ -0,0 +1,32 @@ +version: 2 + +models: + - name: test_executions + description: Base model for metadata about test executions. One row per test execution. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' + - name: failures + description: '{{ doc("failures") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: query_completed_at + description: '{{ doc("query_completed_at") }}' + - name: rows_affected + description: '{{ doc("rows_affected") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: status + description: '{{ doc("status") }}' + - name: test_execution_id + description: '{{ doc("test_execution_id") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' + - name: was_full_refresh + description: '{{ doc("was_full_refresh") }}' + - name: message + description: '{{ doc("message") }}' diff --git a/models/sources/tests.yml b/models/sources/tests.yml new file mode 100644 index 00000000..e4b906bf --- /dev/null +++ b/models/sources/tests.yml @@ -0,0 +1,24 @@ +version: 2 + +models: + - name: tests + description: Base model for metadata about test executions. One row per node per run. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: depends_on_nodes + description: '{{ doc("depends_on_nodes") }}' + - name: name + description: '{{ doc("name") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: package_name + description: '{{ doc("package_name") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: tags + description: '{{ doc("tags") }}' + - name: test_execution_id + description: '{{ doc("test_execution_id") }}' + - name: test_path + description: '{{ doc("test_path") }}' From 3e2dea79dd80fb61622cff3e9bf2ddf28707dd67 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 5 May 2023 13:20:41 +0100 Subject: [PATCH 2/6] Small change to fix nit and give new commit sha --- integration_test_project/models/tests_and_exposures.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_test_project/models/tests_and_exposures.yml b/integration_test_project/models/tests_and_exposures.yml index b397edc0..db0880e2 100644 --- a/integration_test_project/models/tests_and_exposures.yml +++ b/integration_test_project/models/tests_and_exposures.yml @@ -10,7 +10,7 @@ models: - unique exposures: - - name: "ceo's imaginary dashboard" + - name: ceo_imaginary_dashboard type: dashboard maturity: high description: "ceo's favourite dashboard" @@ -23,7 +23,7 @@ exposures: name: Claire from Data email: data@jaffleshop.com - - name: "CIO dashboard" + - name: cio_dashboard type: dashboard maturity: high description: '{{ doc("clickstream") }}' From 2faafa7714373616e7c0ac3ca4a7ad3cc04e44a9 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 12 May 2023 13:15:42 +0100 Subject: [PATCH 3/6] Fix missing and extra fields --- models/sources/exposures.yml | 28 +++++++++---------- models/sources/model_executions.yml | 38 ++++++++++++++------------ models/sources/models.yml | 28 +++++++++---------- models/sources/seed_executions.yml | 34 ++++++++++++----------- models/sources/seeds.yml | 22 ++++++++------- models/sources/snapshot_executions.yml | 34 ++++++++++++----------- models/sources/snapshots.yml | 24 ++++++++-------- models/sources/sources.yml | 26 ++++++++---------- models/sources/test_executions.yml | 26 ++++++++---------- models/sources/tests.yml | 18 ++++++------ 10 files changed, 140 insertions(+), 138 deletions(-) diff --git a/models/sources/exposures.yml b/models/sources/exposures.yml index 273a1327..1ba99420 100644 --- a/models/sources/exposures.yml +++ b/models/sources/exposures.yml @@ -6,25 +6,25 @@ models: columns: - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' - - name: depends_on_nodes - description: '{{ doc("depends_on_nodes") }}' - - name: description - description: '{{ doc("description") }}' - - name: maturity - description: '{{ doc("maturity") }}' - - name: name - description: '{{ doc("name") }}' - name: node_id description: '{{ doc("node_id") }}' - - name: owner - description: '{{ doc("owner") }}' - - name: package_name - description: '{{ doc("package_name") }}' - - name: path - description: '{{ doc("path") }}' - name: run_started_at description: '{{ doc("run_started_at") }}' + - name: name + description: '{{ doc("name") }}' - name: type description: '{{ doc("type") }}' + - name: owner + description: '{{ doc("owner") }}' + - name: maturity + description: '{{ doc("maturity") }}' + - name: path + description: '{{ doc("path") }}' + - name: description + description: '{{ doc("description") }}' - name: url description: '{{ doc("url") }}' + - name: package_name + description: '{{ doc("package_name") }}' + - name: depends_on_nodes + description: '{{ doc("depends_on_nodes") }}' diff --git a/models/sources/model_executions.yml b/models/sources/model_executions.yml index 65b7368a..7ea415fa 100644 --- a/models/sources/model_executions.yml +++ b/models/sources/model_executions.yml @@ -6,33 +6,35 @@ models: columns: - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' - - name: compile_started_at - description: '{{ doc("compile_started_at") }}' - - name: materialization - description: '{{ doc("materialization") }}' - - name: name - description: '{{ doc("name") }}' - - name: alias - description: '{{ doc("alias") }}' - name: node_id description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: was_full_refresh + description: '{{ doc("was_full_refresh") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: status + description: '{{ doc("status") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' - name: query_completed_at description: '{{ doc("query_completed_at") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' - name: rows_affected description: '{{ doc("rows_affected") }}' + # BigQuery only - name: bytes_processed description: '{{ doc("bytes_processed") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' + ################# + - name: materialization + description: '{{ doc("materialization") }}' - name: schema description: '{{ doc("schema") }}' - - name: status - description: '{{ doc("status") }}' - - name: thread_id - description: '{{ doc("thread_id") }}' - - name: total_node_runtime - description: '{{ doc("total_node_runtime") }}' - - name: was_full_refresh - description: '{{ doc("was_full_refresh") }}' + - name: name + description: '{{ doc("name") }}' + - name: alias + description: '{{ doc("alias") }}' - name: message description: '{{ doc("message") }}' diff --git a/models/sources/models.yml b/models/sources/models.yml index 3e77aa79..20dd16ea 100644 --- a/models/sources/models.yml +++ b/models/sources/models.yml @@ -4,31 +4,31 @@ models: - name: models description: Staging model that contains metadata about model executions. One row per node per run. columns: - - name: checksum - description: '{{ doc("checksum") }}' - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' - name: database description: '{{ doc("database") }}' - - name: depends_on_nodes - description: '{{ doc("depends_on_nodes") }}' - - name: materialization - description: '{{ doc("materialization") }}' + - name: schema + description: '{{ doc("schema") }}' - name: name description: '{{ doc("name") }}' - - name: alias - description: '{{ doc("alias") }}' - - name: node_id - description: '{{ doc("node_id") }}' + - name: depends_on_nodes + description: '{{ doc("depends_on_nodes") }}' - name: package_name description: '{{ doc("package_name") }}' - name: path description: '{{ doc("path") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: schema - description: '{{ doc("schema") }}' + - name: checksum + description: '{{ doc("checksum") }}' + - name: materialization + description: '{{ doc("materialization") }}' - name: tags description: '{{ doc("tags") }}' - name: meta description: '{{ doc("meta") }}' + - name: alias + description: '{{ doc("alias") }}' diff --git a/models/sources/seed_executions.yml b/models/sources/seed_executions.yml index c6417dc8..6485363f 100644 --- a/models/sources/seed_executions.yml +++ b/models/sources/seed_executions.yml @@ -6,29 +6,31 @@ models: columns: - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' - - name: compile_started_at - description: '{{ doc("compile_started_at") }}' - - name: materialization - description: '{{ doc("materialization") }}' - - name: name - description: '{{ doc("name") }}' - - name: alias - description: '{{ doc("alias") }}' - name: node_id description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: was_full_refresh + description: '{{ doc("was_full_refresh") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: status + description: '{{ doc("status") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' - name: query_completed_at description: '{{ doc("query_completed_at") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' - name: rows_affected description: '{{ doc("rows_affected") }}' + - name: materialization + description: '{{ doc("materialization") }}' - name: schema description: '{{ doc("schema") }}' - - name: status - description: '{{ doc("status") }}' - - name: thread_id - description: '{{ doc("thread_id") }}' - - name: total_node_runtime - description: '{{ doc("total_node_runtime") }}' - - name: was_full_refresh - description: '{{ doc("was_full_refresh") }}' + - name: name + description: '{{ doc("name") }}' + - name: alias + description: '{{ doc("alias") }}' - name: message description: '{{ doc("message") }}' diff --git a/models/sources/seeds.yml b/models/sources/seeds.yml index 46723eec..8f19dd19 100644 --- a/models/sources/seeds.yml +++ b/models/sources/seeds.yml @@ -4,23 +4,25 @@ models: - name: seeds description: Base model for metadata about seed executions. One row per node per run. columns: - - name: checksum - description: '{{ doc("checksum") }}' - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' - name: database description: '{{ doc("database") }}' + - name: schema + description: '{{ doc("schema") }}' - name: name description: '{{ doc("name") }}' - - name: alias - description: '{{ doc("alias") }}' - - name: node_id - description: '{{ doc("node_id") }}' - name: package_name description: '{{ doc("package_name") }}' - name: path description: '{{ doc("path") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: schema - description: '{{ doc("schema") }}' + - name: checksum + description: '{{ doc("checksum") }}' + - name: meta + description: '{{ doc("meta") }}' + - name: alias + description: '{{ doc("alias") }}' diff --git a/models/sources/snapshot_executions.yml b/models/sources/snapshot_executions.yml index 5a5ce560..a6696954 100644 --- a/models/sources/snapshot_executions.yml +++ b/models/sources/snapshot_executions.yml @@ -6,29 +6,31 @@ models: columns: - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' - - name: compile_started_at - description: '{{ doc("compile_started_at") }}' - - name: materialization - description: '{{ doc("materialization") }}' - - name: name - description: '{{ doc("name") }}' - - name: alias - description: '{{ doc("alias") }}' - name: node_id description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: was_full_refresh + description: '{{ doc("was_full_refresh") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: status + description: '{{ doc("status") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' - name: query_completed_at description: '{{ doc("query_completed_at") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' - name: rows_affected description: '{{ doc("rows_affected") }}' + - name: materialization + description: '{{ doc("materialization") }}' - name: schema description: '{{ doc("schema") }}' - - name: status - description: '{{ doc("status") }}' - - name: thread_id - description: '{{ doc("thread_id") }}' - - name: total_node_runtime - description: '{{ doc("total_node_runtime") }}' - - name: was_full_refresh - description: '{{ doc("was_full_refresh") }}' + - name: name + description: '{{ doc("name") }}' + - name: alias + description: '{{ doc("alias") }}' - name: message description: '{{ doc("message") }}' diff --git a/models/sources/snapshots.yml b/models/sources/snapshots.yml index b78b2f8a..b33c8242 100644 --- a/models/sources/snapshots.yml +++ b/models/sources/snapshots.yml @@ -4,29 +4,29 @@ models: - name: snapshots description: Base model for metadata about seed executions. One row per node per run. columns: - - name: checksum - description: '{{ doc("checksum") }}' - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' - name: database description: '{{ doc("database") }}' - - name: depends_on_nodes - description: '{{ doc("depends_on_nodes") }}' + - name: schema + description: '{{ doc("schema") }}' - name: name description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' + - name: depends_on_nodes + description: '{{ doc("depends_on_nodes") }}' - name: package_name description: '{{ doc("package_name") }}' - name: path description: '{{ doc("path") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: schema - description: '{{ doc("schema") }}' - - name: snapshot_execution_id - description: '{{ doc("snapshot_execution_id") }}' + - name: checksum + description: '{{ doc("checksum") }}' - name: strategy description: '{{ doc("strategy") }}' + - name: meta + description: '{{ doc("meta") }}' - name: alias description: '{{ doc("alias") }}' diff --git a/models/sources/sources.yml b/models/sources/sources.yml index b538ce8f..15f8f4a8 100644 --- a/models/sources/sources.yml +++ b/models/sources/sources.yml @@ -6,25 +6,23 @@ models: columns: - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' - - name: database - description: '{{ doc("database") }}' - - name: freshness - description: '{{ doc("freshness") }}' - - name: identifier - description: '{{ doc("identifier") }}' - - name: loaded_at_field - description: '{{ doc("loaded_at_field") }}' - - name: loader - description: '{{ doc("loader") }}' - - name: name - description: '{{ doc("name") }}' - name: node_id description: '{{ doc("node_id") }}' - name: run_started_at description: '{{ doc("run_started_at") }}' + - name: database + description: '{{ doc("database") }}' - name: schema description: '{{ doc("schema") }}' - - name: source_execution_id - description: '{{ doc("source_execution_id") }}' - name: source_name description: '{{ doc("source_name") }}' + - name: loader + description: '{{ doc("loader") }}' + - name: name + description: '{{ doc("name") }}' + - name: identifier + description: '{{ doc("identifier") }}' + - name: loaded_at_field + description: '{{ doc("loaded_at_field") }}' + - name: freshness + description: '{{ doc("freshness") }}' diff --git a/models/sources/test_executions.yml b/models/sources/test_executions.yml index 8a76de5c..1f7e07e7 100644 --- a/models/sources/test_executions.yml +++ b/models/sources/test_executions.yml @@ -6,27 +6,25 @@ models: columns: - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' - - name: compile_started_at - description: '{{ doc("compile_started_at") }}' - - name: failures - description: '{{ doc("failures") }}' - name: node_id description: '{{ doc("node_id") }}' - - name: query_completed_at - description: '{{ doc("query_completed_at") }}' - - name: rows_affected - description: '{{ doc("rows_affected") }}' - name: run_started_at description: '{{ doc("run_started_at") }}' - - name: status - description: '{{ doc("status") }}' - - name: test_execution_id - description: '{{ doc("test_execution_id") }}' + - name: was_full_refresh + description: '{{ doc("was_full_refresh") }}' - name: thread_id description: '{{ doc("thread_id") }}' + - name: status + description: '{{ doc("status") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' + - name: query_completed_at + description: '{{ doc("query_completed_at") }}' - name: total_node_runtime description: '{{ doc("total_node_runtime") }}' - - name: was_full_refresh - description: '{{ doc("was_full_refresh") }}' + - name: rows_affected + description: '{{ doc("rows_affected") }}' + - name: failures + description: '{{ doc("failures") }}' - name: message description: '{{ doc("message") }}' diff --git a/models/sources/tests.yml b/models/sources/tests.yml index e4b906bf..9d8b8b7b 100644 --- a/models/sources/tests.yml +++ b/models/sources/tests.yml @@ -6,19 +6,17 @@ models: columns: - name: command_invocation_id description: '{{ doc("command_invocation_id") }}' - - name: depends_on_nodes - description: '{{ doc("depends_on_nodes") }}' - - name: name - description: '{{ doc("name") }}' - name: node_id description: '{{ doc("node_id") }}' - - name: package_name - description: '{{ doc("package_name") }}' - name: run_started_at description: '{{ doc("run_started_at") }}' - - name: tags - description: '{{ doc("tags") }}' - - name: test_execution_id - description: '{{ doc("test_execution_id") }}' + - name: name + description: '{{ doc("name") }}' + - name: depends_on_nodes + description: '{{ doc("depends_on_nodes") }}' + - name: package_name + description: '{{ doc("package_name") }}' - name: test_path description: '{{ doc("test_path") }}' + - name: tags + description: '{{ doc("tags") }}' From 9cff9757a4bd0f11791376052c43e9eaa8f2a3f3 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 12 May 2023 13:22:29 +0100 Subject: [PATCH 4/6] Remove field that doesn't exist everywhere --- models/sources/model_executions.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/models/sources/model_executions.yml b/models/sources/model_executions.yml index 7ea415fa..f4968eba 100644 --- a/models/sources/model_executions.yml +++ b/models/sources/model_executions.yml @@ -24,10 +24,9 @@ models: description: '{{ doc("total_node_runtime") }}' - name: rows_affected description: '{{ doc("rows_affected") }}' - # BigQuery only - - name: bytes_processed - description: '{{ doc("bytes_processed") }}' - ################# + # BigQuery only - dbx requires all documented fields to exist in the model + # - name: bytes_processed + # description: '{{ doc("bytes_processed") }}' - name: materialization description: '{{ doc("materialization") }}' - name: schema From 1a831fb06b61677f582584969ce7b27eb23d4787 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 12 May 2023 13:31:05 +0100 Subject: [PATCH 5/6] Change environment variables --- .github/workflows/main_test_package.yml | 4 +++- .github/workflows/publish_docs_on_release.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 2f72384d..4f7217e6 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -24,7 +24,9 @@ env: DBT_CLOUD_PROJECT_ID: 123 DBT_CLOUD_JOB_ID: ABC DBT_CLOUD_RUN_REASON: "String with 'quotes' !" - TEST_ENV_VAR_1: TEST_VALUE + TEST_ENV_VAR_NUMBER: 3 + TEST_ENV_VAR_EMPTY: "" + TEST_ENV_VAR_WITH_QUOTE: "Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG." DBT_ENV_CUSTOM_ENV_FAVOURITE_DBT_PACKAGE: dbt_artifacts jobs: diff --git a/.github/workflows/publish_docs_on_release.yml b/.github/workflows/publish_docs_on_release.yml index 49b1ab00..cb588373 100644 --- a/.github/workflows/publish_docs_on_release.yml +++ b/.github/workflows/publish_docs_on_release.yml @@ -20,7 +20,9 @@ env: DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }} DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }} # Env vars to test invocations model - TEST_ENV_VAR_1: TEST_VALUE + TEST_ENV_VAR_NUMBER: 3 + TEST_ENV_VAR_EMPTY: "" + TEST_ENV_VAR_WITH_QUOTE: "Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG." jobs: generate-dbt-docs: From 0a3f72594f05c6da463d030781b30c2042a2b42c Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 12 May 2023 13:43:52 +0100 Subject: [PATCH 6/6] Add back in TEST_ENV_VAR for old release --- .github/workflows/ci_test_package.yml | 1 + .github/workflows/main_test_package.yml | 1 + tox.ini | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index f1db448e..7cb8ce95 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -24,6 +24,7 @@ env: DBT_CLOUD_PROJECT_ID: 123 DBT_CLOUD_JOB_ID: ABC DBT_CLOUD_RUN_REASON: "String with 'quotes' !" + TEST_ENV_VAR_1: TEST_VALUE TEST_ENV_VAR_NUMBER: 3 TEST_ENV_VAR_EMPTY: "" TEST_ENV_VAR_WITH_QUOTE: "Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG." diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 4f7217e6..ec1f173e 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -24,6 +24,7 @@ env: DBT_CLOUD_PROJECT_ID: 123 DBT_CLOUD_JOB_ID: ABC DBT_CLOUD_RUN_REASON: "String with 'quotes' !" + TEST_ENV_VAR_1: TEST_VALUE TEST_ENV_VAR_NUMBER: 3 TEST_ENV_VAR_EMPTY: "" TEST_ENV_VAR_WITH_QUOTE: "Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG." diff --git a/tox.ini b/tox.ini index 9d5ae1d5..db7d8443 100644 --- a/tox.ini +++ b/tox.ini @@ -90,6 +90,7 @@ passenv = DBT_CLOUD_RUN_ID DBT_CLOUD_RUN_REASON_CATEGORY DBT_CLOUD_RUN_REASON + TEST_ENV_VAR_1 TEST_ENV_VAR_NUMBER TEST_ENV_VAR_EMPTY TEST_ENV_VAR_WITH_QUOTE