Merge pull request #65 from dbt-labs/clean_adding_pytest

Adding Pytest Tests

.github/CODEOWNERS

@@ -1 +1,2 @@
 @joellabes
+@callum-mcdata

.github/actions/end-to-end-test/action.yml

@@ -1,37 +1,28 @@
 name: "End to end testing"
 description: "Set up profile and run dbt with test project"
 inputs:
-  dbt-project:
-    description: "Location of test project"
-    required: false
-    default: "integration_tests"
-  dbt-target:
+  dbt_target:
     description: "Name of target to use when running dbt"
     required: true
-  database-adapter-package:
-    description: "Name of database adapter to install"
-    required: true
+
 
 runs:
   using: "composite"
   steps:
+
     - name: Install python dependencies
       shell: bash
       run: |
         pip install --user --upgrade pip
         pip --version
-        pip install -r dev-requirements.txt
-
-    - name: Setup dbt profile
-      shell: bash
-      run: |
-        mkdir -p $HOME/.dbt
-        cp ${{ github.action_path }}/sample.profiles.yml $HOME/.dbt/profiles.yml
+        pip install -r dev-requirements.txt        
 
-    - name: Run dbt
+    ## Make sure to defined dbt_target as an environment variable
+    ## Previously we were supplying just as an input and os.environ
+    ## wasn't able to recognize it.
+    - name: Run pytest
       shell: bash
+      env:
+        dbt_target: ${{ inputs.dbt_target }}
       run: |
-        cd ${{ inputs.dbt-project }}
-        dbt deps --target ${{ inputs.dbt-target }}
-        dbt seed --target ${{ inputs.dbt-target }} --full-refresh
-        dbt build --target ${{ inputs.dbt-target }} --full-refresh
+        python3 -m pytest tests/functional

.github/workflows/ci.yml

@@ -29,6 +29,8 @@ jobs:
           POSTGRES_USER: ${{ env.POSTGRES_TEST_USER }}
           POSTGRES_PASSWORD: ${{ env.POSTGRES_TEST_PASSWORD }}
           POSTGRES_DB: ${{ env.POSTGRES_TEST_DB }}
+          POSTGRES_TEST_PORT: ${{ env.POSTGRES_TEST_PORT }}
+          POSTGRES_TEST_HOST: ${{ env.POSTGRES_TEST_HOST }}
         ports:
           - 5432:5432
         options: >-
@@ -48,11 +50,10 @@ jobs:
 
       - uses: ./.github/actions/end-to-end-test
         with:
-          dbt-target: postgres
-          database-adapter-package: dbt-postgres
+          dbt_target: postgres
 
   snowflake:
-    # needs: postgres
+    needs: postgres
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
@@ -72,11 +73,10 @@ jobs:
           SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
           SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
         with:
-          dbt-target: snowflake
-          database-adapter-package: dbt-snowflake
+          dbt_target: snowflake
 
   redshift:
-    # needs: postgres
+    needs: postgres
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
@@ -95,11 +95,10 @@ jobs:
           REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
           REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
         with:
-          dbt-target: redshift
-          database-adapter-package: dbt-redshift
+          dbt_target: redshift
 
   bigquery:
-    # needs: postgres
+    needs: postgres
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
@@ -119,11 +118,10 @@ jobs:
           KEYFILE_PATH=$HOME/.dbt/bigquery-service-key.json
           echo $BIGQUERY_TEST_SERVICE_ACCOUNT_JSON > $KEYFILE_PATH
           echo ::set-output name=path::$KEYFILE_PATH
-
       - uses: ./.github/actions/end-to-end-test
         env:
           BIGQUERY_SERVICE_KEY_PATH: ${{ steps.keyfile.outputs.path }}
           BIGQUERY_TEST_DATABASE: ${{ secrets.BIGQUERY_TEST_DATABASE }}
+          BIGQUERY_TEST_PROJECT: ${{ secrets.BIGQUERY_TEST_PROJECT }}
         with:
-          dbt-target: bigquery
-          database-adapter-package: dbt-bigquery
+          dbt_target: bigquery

.gitignore

@@ -5,4 +5,11 @@ dbt_packages/
 logs/
 .DS_Store
 integration_tests/target
-model_testing/
+model_testing/
+test.env
+env/
+.venv
+venv/
+.env
+__pycache__
+.pytest_cache

CONTRIBUTING.md

@@ -90,6 +90,70 @@ If you are a member of the `dbt-labs` GitHub organization, you will have push ac
 
 When you create a Pull Request (below), integration tests will automatically run. When you add new functionality or change existing functionality, please also add new tests to ensure that the project remains resilient.
 
+## Testing locally
+
+### Initial setup
+
+Postgres offers the easiest way to test most functionality today. To run the Postgres integration tests, you'll have to do one extra step of setting up the test database:
+
+```shell
+docker-compose up -d database
+```
+
+### Virtual environment
+
+If you are using a shell other than `zsh` or `bash`, you will need to [adjust the `activate` commands](https://docs.python.org/3/library/venv.html) below accordingly.
+
+<details>
+  <summary>Platform-specific instructions for venv activation</summary>
+
+| **Platform** | **Shell**       | **Command to activate virtual environment** |
+|--------------|-----------------|---------------------------------------------|
+| POSIX        | bash/zsh        | `$ source env/bin/activate`                 |
+|              | fish            | `$ source env/bin/activate.fish`            |
+|              | csh/tcsh        | `$ source env/bin/activate.csh`             |
+|              | PowerShell Core | `$ env/bin/Activate.ps1`                    |
+| Windows      | cmd.exe         | `C:\> env\Scripts\activate.bat`             |
+|              | PowerShell      | `PS C:\> env\Scripts\Activate.ps1`          |
+
+</details>
+
+```shell
+python -m venv env
+source env/bin/activate
+python3 -m pip install --upgrade pip
+python3 -m pip install --pre -r dev-requirements.txt
+source env/bin/activate
+```
+
+You can run integration tests "locally" by configuring a `test.env` file with appropriate environment variables.
+
+```
+cp test.env.example test.env
+$EDITOR test.env
+```
+
+WARNING: The `test.env` file you create is `.gitignore`'d, but please be _extra_ careful to never check in credentials or other sensitive information when developing.
+
+### Test commands
+
+There are many options for invoking `pytest` and choosing which tests to execute. See [here](https://docs.pytest.org/usage.html) for the `pytest` documentation. Some common options are included below.
+
+#### Run all the tests
+```shell
+python3 -m pytest
+```
+
+#### Run tests in a module
+```shell
+python3 -m pytest tests/functional/example/test_example_failing_test.py
+```
+
+#### Run tests in a directory
+```shell
+python3 -m pytest tests/functional
+```
+
 ## Submitting a Pull Request
 
 A `dbt_metrics` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.

dev-requirements.txt

@@ -1,4 +1,17 @@
-git+https://github.com/dbt-labs/dbt-redshift.git@c35865b
-git+https://github.com/dbt-labs/dbt-snowflake.git@a017382
-git+https://github.com/dbt-labs/dbt-bigquery.git@d52222e
-git+https://github.com/dbt-labs/dbt-core.git@0db634d#egg=dbt-core&subdirectory=core
+pytest
+pytest-dotenv
+
+# Bleeding edge
+# git+https://github.com/dbt-labs/dbt-core.git@main#egg=dbt-tests-adapter&subdirectory=tests/adapter
+# git+https://github.com/dbt-labs/dbt-core.git@main#egg=dbt-core&subdirectory=core
+# git+https://github.com/dbt-labs/dbt-core.git@main#egg=dbt-postgres&subdirectory=plugins/postgres
+# git+https://github.com/dbt-labs/dbt-redshift.git
+# git+https://github.com/dbt-labs/dbt-snowflake.git
+# git+https://github.com/dbt-labs/dbt-bigquery.git
+
+# Most recent release candidates
+dbt-tests-adapter==1.2.0
+dbt-core==1.2.0
+dbt-redshift==1.2.0
+dbt-snowflake==1.2.0
+dbt-bigquery==1.2.0

docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3.5"
+services:
+  database:
+    image: postgres
+    environment:
+      POSTGRES_USER: "root"
+      POSTGRES_PASSWORD: "password"
+      POSTGRES_DB: "dbt"
+    ports:
+      - "5432:5432"
+
+networks:
+  default:
+    name: dbt-net

integration_tests/dbt_project.yml

@@ -6,7 +6,7 @@ version: "1.0.0"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project.
-profile: "dbt_metrics_integration_tests"
+profile: "dbt_metrics_integration_tests_bigquery"
 
 model-paths: ["models"]
 analysis-paths: ["analyses"]

integration_tests/models/metric_definitions/base_average_metric.yml

@@ -4,7 +4,7 @@ metrics:
     model: ref('fact_orders')
     label: Total Discount ($)
     timestamp: order_date
-    time_grains: [day, week]
+    time_grains: [day, week, month]
     type: average
     sql: discount_total
     dimensions:

integration_tests/models/metric_definitions/base_count_metric.yml

@@ -1,14 +1,12 @@
 version: 2 
 metrics:
-    - name: base_count_metric
-      label: Members of Community Slack
-      model: ref('seed_slack_users')
-
-      type: count
-      sql: "*"
-      timestamp: joined_at
-      time_grains: [day, week, month] 
-
-      dimensions: 
-        - is_active_past_quarter
-        - has_messaged
+  - name: base_count_metric
+    model: ref('fact_orders')
+    label: Total Discount ($)
+    timestamp: order_date
+    time_grains: [day, week, month]
+    type: count
+    sql: order_total
+    dimensions:
+      - had_discount
+      - order_country

integration_tests/models/metric_definitions/base_sum_metric.yml

@@ -4,7 +4,7 @@ metrics:
     model: ref('fact_orders')
     label: Order Total ($)
     timestamp: order_date
-    time_grains: [day, week]
+    time_grains: [day, week,month]
     type: sum
     sql: order_total
     dimensions:

integration_tests/models/metric_definitions/expression_metric.yml

@@ -14,7 +14,7 @@ metrics:
   - name: ratio_metric
     label: Ratio ($)
     timestamp: order_date
-    time_grains: [day, week]
+    time_grains: [day, week,month]
     type: expression
     sql: "{{metric('base_sum_metric')}} / {{metric('base_average_metric')}}"
     dimensions:

integration_tests/models/metric_testing_models/base_average_metric.sql

@@ -1,6 +1,6 @@
 select *
 from 
 {{ metrics.calculate(metric('base_average_metric'), 
-    grain='day', 
+    grain='month', 
     dimensions=['had_discount']) 
 }}

integration_tests/models/metric_testing_models/base_count_distinct_metric.sql

@@ -0,0 +1,6 @@
+select *
+from 
+{{ metrics.calculate(metric('base_count_distinct_metric'), 
+    grain='month'
+    ) 
+}}

integration_tests/models/metric_testing_models/base_count_distinct_metric.yml

@@ -0,0 +1,18 @@
+version: 2 
+models:
+  - name: base_count_distinct_metric
+    tests: 
+      - dbt_utils.equality:
+          compare_model: ref('base_count_distinct_metric__expected')
+
+metrics:
+  - name: base_count_distinct_metric
+    model: ref('fact_orders')
+    label: Count Distinct
+    timestamp: order_date
+    time_grains: [day, week, month]
+    type: count_distinct
+    sql: customer_id
+    dimensions:
+      - had_discount
+      - order_country