From f1bdd5d8afaee8e8139f14926390ad3d8cfb2ef3 Mon Sep 17 00:00:00 2001 From: 0adamjones <105521832+0adamjones@users.noreply.github.com> Date: Thu, 2 Feb 2023 21:06:14 -0800 Subject: [PATCH 01/12] update readme, remove bq and redshift from ci job --- .circleci/config.yml | 27 +-------------------------- README.md | 4 ++-- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 72f6493..0f0f8be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: . dbt_venv/bin/activate pip install --upgrade pip setuptools - pip install --pre dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery + pip install --pre dbt-core dbt-postgres dbt-snowflake mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml @@ -48,17 +48,6 @@ jobs: dbt --warn-error run --target postgres - - run: - name: "Run Tests - Redshift" - command: | - . dbt_venv/bin/activate - echo `pwd` - cd integration_tests - dbt --warn-error deps - dbt --warn-error seed --target redshift - dbt --warn-error run --target redshift --full-refresh - dbt --warn-error run --target redshift - - run: name: "Run Tests - Snowflake" command: | @@ -70,20 +59,6 @@ jobs: dbt --warn-error run --target snowflake --full-refresh dbt --warn-error run --target snowflake - - run: - name: "Run Tests - BigQuery" - environment: - BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" - - command: | - . dbt_venv/bin/activate - echo `pwd` - cd integration_tests - dbt --warn-error deps - dbt --warn-error seed --target bigquery - dbt --warn-error run --target bigquery --full-refresh - dbt --warn-error run --target bigquery - - save_cache: key: deps1-{{ .Branch }} paths: diff --git a/README.md b/README.md index 9362ac5..a6726af 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -> ⛔🏚️ This package is no longer developed or maintained by dbt Labs. If you're interested in forking/adopting it, reach out in #package-ecosystem on dbt Slack. +> ⛔🏚️ This package is not officially related to Segment and is maintained by [Fleetio](https://fleetio.com). CI jobs run on PRs will only test for postgres and Snowflake compatibility. Please submit an issue for any bugs/feature requests related to Redshift or BigQuery and we'll figure out how to help! # dbt-segment This [dbt package](https://docs.getdbt.com/docs/package-management): @@ -54,4 +54,4 @@ the package to run successfully. This package has been tested on Redshift, Snowflake, BigQuery, and Postgres. ### Contributing -Additional contributions to this repo are very welcome! Check out [this post](https://discourse.getdbt.com/t/contributing-to-a-dbt-package/657) on the best workflow for contributing to a package. All PRs should only include functionality that is contained within all Segment deployments; no implementation-specific details should be included. +Additional contributions to this repo are very welcome! Check out [this post](https://discourse.getdbt.com/t/contributing-to-a-dbt-package/657) on the best workflow for contributing to a package. All PRs should only include functionality that is contained within all Segment deployments; no implementation-specific details should be included. CI jobs run on PRs will only test for postgres and Snowflake compatibility. From cb998198269e03c5e3999725b4a69ecfbeaedcfb Mon Sep 17 00:00:00 2001 From: 0adamjones <105521832+0adamjones@users.noreply.github.com> Date: Fri, 3 Feb 2023 13:25:29 -0800 Subject: [PATCH 02/12] commenting out bigquery variable --- .circleci/config.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f0f8be..4d7f55a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,10 +10,11 @@ jobs: steps: - checkout - - run: - run: setup_creds - command: | - echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json + # - run: + # run: setup_creds + # command: | + # echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json + - restore_cache: key: deps1-{{ .Branch }} From 8ff87edcc03cab622f01eeb420da3906c98b9525 Mon Sep 17 00:00:00 2001 From: 0adamjones <105521832+0adamjones@users.noreply.github.com> Date: Fri, 3 Feb 2023 13:32:27 -0800 Subject: [PATCH 03/12] adding comments --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d7f55a..32da717 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,10 @@ jobs: steps: - checkout + # fleetio note: + # the section below was written when dbt labs owned the package and supported bigquery and redshift + # we don't have test environments for redshift and bigquery, but we wanted to keep this in case we can support it in the future + # - run: # run: setup_creds # command: | From fb829506882ad13440a1cabbfefd2b5363076c64 Mon Sep 17 00:00:00 2001 From: jchen Date: Thu, 9 Feb 2023 21:53:37 -0600 Subject: [PATCH 04/12] fix column quoting in integration tests' --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 9b78afd..2661b4c 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -11,4 +11,4 @@ vars: segment_page_views_table: "{{ ref('example_segment_pages') }}" seeds: - +quote_columns: false + +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}" From b4f81c97de90a693d785ffed76bca5177ec8d821 Mon Sep 17 00:00:00 2001 From: jchen Date: Thu, 9 Feb 2023 22:01:31 -0600 Subject: [PATCH 05/12] pin dbt to our version --- .circleci/config.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 32da717..7aac667 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,7 @@ jobs: . dbt_venv/bin/activate pip install --upgrade pip setuptools - pip install --pre dbt-core dbt-postgres dbt-snowflake + pip install --pre dbt-core==1.3.2 dbt-postgres==1.3.2 dbt-snowflake==1.3.0 mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2661b4c..3448b8e 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -8,7 +8,7 @@ profile: 'integration_tests' vars: segment: - segment_page_views_table: "{{ ref('example_segment_pages') }}" + segment_page_views_table: example_segment_pages seeds: +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}" From 6adde7cf7ba0ad7e4039c3e81528a399959dbf15 Mon Sep 17 00:00:00 2001 From: jchen Date: Thu, 9 Feb 2023 22:02:04 -0600 Subject: [PATCH 06/12] try using seed ref --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 3448b8e..2661b4c 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -8,7 +8,7 @@ profile: 'integration_tests' vars: segment: - segment_page_views_table: example_segment_pages + segment_page_views_table: "{{ ref('example_segment_pages') }}" seeds: +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}" From a760b7691658827e2c27e0bcee9d73359a452ffa Mon Sep 17 00:00:00 2001 From: jchen Date: Thu, 9 Feb 2023 22:04:26 -0600 Subject: [PATCH 07/12] revert to latest version and hardcode table name in variable --- .circleci/config.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7aac667..32da717 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,7 @@ jobs: . dbt_venv/bin/activate pip install --upgrade pip setuptools - pip install --pre dbt-core==1.3.2 dbt-postgres==1.3.2 dbt-snowflake==1.3.0 + pip install --pre dbt-core dbt-postgres dbt-snowflake mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2661b4c..3448b8e 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -8,7 +8,7 @@ profile: 'integration_tests' vars: segment: - segment_page_views_table: "{{ ref('example_segment_pages') }}" + segment_page_views_table: example_segment_pages seeds: +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}" From c8a91b3a632f6ec1da0a9d6e0741baa39c3940a9 Mon Sep 17 00:00:00 2001 From: jchen Date: Thu, 9 Feb 2023 22:19:49 -0600 Subject: [PATCH 08/12] syntax fix for package name --- integration_tests/dbt_project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 3448b8e..8ddd8ac 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,8 +7,8 @@ seed-paths: ["seeds"] profile: 'integration_tests' vars: - segment: - segment_page_views_table: example_segment_pages + dbt-segment: + segment_page_views_table: "{{ ref('example_segment_pages') }}" seeds: +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}" From ac40e5c1a5d5924aabc03762861c5eae1aea05f0 Mon Sep 17 00:00:00 2001 From: jchen Date: Thu, 9 Feb 2023 22:21:50 -0600 Subject: [PATCH 09/12] typo --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 8ddd8ac..88eef3c 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,7 @@ seed-paths: ["seeds"] profile: 'integration_tests' vars: - dbt-segment: + dbt_segment: segment_page_views_table: "{{ ref('example_segment_pages') }}" seeds: From 8525c26eb425e52c9e3abe92106785227b2645ab Mon Sep 17 00:00:00 2001 From: jchen Date: Thu, 9 Feb 2023 22:35:38 -0600 Subject: [PATCH 10/12] add in new column to seed --- integration_tests/seeds/example_segment_pages.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/seeds/example_segment_pages.csv b/integration_tests/seeds/example_segment_pages.csv index d8eef2b..2cacef9 100644 --- a/integration_tests/seeds/example_segment_pages.csv +++ b/integration_tests/seeds/example_segment_pages.csv @@ -1,2 +1,2 @@ -id,anonymous_id,user_id,received_at,sent_at,timestamp,url,path,title,search,referrer,context_campaign_source,context_campaign_medium,context_campaign_name,context_campaign_term,context_campaign_content,context_ip,context_user_agent -ajs-9527e97fc714abe02a23b3d24cf09a36,507f191e810c19729de860ea,97980cfea0067,2015-02-23 22:28:55,2015-02-23 22:28:55,2015-02-23 22:28:55,https://www.getdbt.com/,/product,Product,?name=ferret,https://www.google.com/,facebook,social,autumn_collection_2015,foo,bar,8.8.8.8,"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36" +id,anonymous_id,user_id,received_at,sent_at,timestamp,url,path,title,search,referrer,context_campaign_source,context_campaign_medium,context_campaign_name,context_campaign_term,context_campaign_content,context_ip,context_user_agent,context_campaign_type +ajs-9527e97fc714abe02a23b3d24cf09a36,507f191e810c19729de860ea,97980cfea0067,2015-02-23 22:28:55,2015-02-23 22:28:55,2015-02-23 22:28:55,https://www.getdbt.com/,/product,Product,?name=ferret,https://www.google.com/,facebook,social,autumn_collection_2015,foo,bar,8.8.8.8,"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36","typein" From cd681c2eb682eda9e54a910c1e1bed6441ef527e Mon Sep 17 00:00:00 2001 From: 0adamjones <105521832+0adamjones@users.noreply.github.com> Date: Fri, 10 Feb 2023 08:37:32 -0800 Subject: [PATCH 11/12] Updating CI config, readme --- .circleci/config.yml | 46 ++++++++++++++++++++++++++++++++++---------- README.md | 4 ++-- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 32da717..efdc8dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,7 @@ jobs: # fleetio note: # the section below was written when dbt labs owned the package and supported bigquery and redshift # we don't have test environments for redshift and bigquery, but we wanted to keep this in case we can support it in the future + # we're working on setting up a snowflake service account to use for the snowflake test, but for now we'll only run the CI job for postgres # - run: # run: setup_creds @@ -51,18 +52,43 @@ jobs: dbt --warn-error seed --target postgres dbt --warn-error run --target postgres --full-refresh dbt --warn-error run --target postgres + + # - run: + # name: "Run Tests - Redshift" + # command: | + # . dbt_venv/bin/activate + # echo `pwd` + # cd integration_tests + # dbt --warn-error deps + # dbt --warn-error seed --target redshift + # dbt --warn-error run --target redshift --full-refresh + # dbt --warn-error run --target redshift + # - run: + # name: "Run Tests - BigQuery" + # environment: + # BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" - - run: - name: "Run Tests - Snowflake" - command: | - . dbt_venv/bin/activate - echo `pwd` - cd integration_tests - dbt --warn-error deps - dbt --warn-error seed --target snowflake - dbt --warn-error run --target snowflake --full-refresh - dbt --warn-error run --target snowflake + # command: | + # . dbt_venv/bin/activate + # echo `pwd` + # cd integration_tests + # dbt --warn-error deps + # dbt --warn-error seed --target bigquery + # dbt --warn-error run --target bigquery --full-refresh + # dbt --warn-error run --target bigquery + + + # - run: + # name: "Run Tests - Snowflake" + # command: | + # . dbt_venv/bin/activate + # echo `pwd` + # cd integration_tests + # dbt --warn-error deps + # dbt --warn-error seed --target snowflake + # dbt --warn-error run --target snowflake --full-refresh + # dbt --warn-error run --target snowflake - save_cache: key: deps1-{{ .Branch }} diff --git a/README.md b/README.md index a6726af..d733d93 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -> ⛔🏚️ This package is not officially related to Segment and is maintained by [Fleetio](https://fleetio.com). CI jobs run on PRs will only test for postgres and Snowflake compatibility. Please submit an issue for any bugs/feature requests related to Redshift or BigQuery and we'll figure out how to help! +> ⛔🏚️ This package is not officially related to Segment and is maintained by [Fleetio](https://fleetio.com). CI jobs run on PRs will only test for postgres compatibility. We're working on setting up a service account to support tests for Snowflake. Please submit an issue for any bugs/feature requests related to Redshift or BigQuery and we'll figure out how to help! # dbt-segment This [dbt package](https://docs.getdbt.com/docs/package-management): @@ -54,4 +54,4 @@ the package to run successfully. This package has been tested on Redshift, Snowflake, BigQuery, and Postgres. ### Contributing -Additional contributions to this repo are very welcome! Check out [this post](https://discourse.getdbt.com/t/contributing-to-a-dbt-package/657) on the best workflow for contributing to a package. All PRs should only include functionality that is contained within all Segment deployments; no implementation-specific details should be included. CI jobs run on PRs will only test for postgres and Snowflake compatibility. +Additional contributions to this repo are very welcome! Check out [this post](https://discourse.getdbt.com/t/contributing-to-a-dbt-package/657) on the best workflow for contributing to a package. All PRs should only include functionality that is contained within all Segment deployments; no implementation-specific details should be included. CI jobs run on PRs will only test for postgres compatibility. From 735e4fe3d4f13b441cc058b1d0370d4c5065fa15 Mon Sep 17 00:00:00 2001 From: jchen Date: Fri, 10 Feb 2023 11:39:04 -0600 Subject: [PATCH 12/12] adding in release drafter for release management --- .github/release-drafter.yml | 45 +++++++++++++++++++++++++++ .github/workflows/release-drafter.yml | 29 +++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 .github/release-drafter.yml create mode 100644 .github/workflows/release-drafter.yml diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 0000000..0715380 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,45 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +template: | + # What's Changed + + $CHANGES + + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION + +categories: + - title: '🚀 New Features' + labels: + - 'feature' + - 'enhancement' + - title: '🐛 Bug Fixes' + labels: + - 'fix' + - 'bugfix' + - 'bug' + - title: '🧰 Maintenance' + labels: + - 'chore' + - 'docs' + - 'security' + - 'dependencies' + +version-resolver: + major: + labels: + - 'major' + minor: + labels: + - 'feature' + - 'enhancement' + patch: + labels: + - 'fix' + - 'bugfix' + - 'bug' + - 'dependencies' + - 'security' + - 'docs' + - 'chore' + default: patch + diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 0000000..54674cc --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,29 @@ +name: Release Drafter + +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + update_release_draft: + permissions: + # write permission is required to create a github release + contents: write + # write permission is required for autolabeler + # otherwise, read permission is required at least + pull-requests: write + runs-on: ubuntu-latest + steps: + + # Drafts your next Release notes as Pull Requests are merged into "main" + - uses: release-drafter/release-drafter@v5 + # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml + # with: + # config-name: my-config.yml + # disable-autolabeler: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file