diff --git a/.circleci/config.yml b/.circleci/config.yml index 51cf1bd1ead..bb0ac29ff83 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2.1 jobs: unit: docker: &test_only - - image: fishtownanalytics/test-container:7 + - image: fishtownanalytics/test-container:9 environment: DBT_INVOCATION_ENV: circle steps: @@ -30,7 +30,7 @@ jobs: destination: dist integration-postgres-py36: docker: &test_and_postgres - - image: fishtownanalytics/test-container:7 + - image: fishtownanalytics/test-container:9 environment: DBT_INVOCATION_ENV: circle - image: postgres diff --git a/.gitignore b/.gitignore index e3ebed6763f..acb15d1f0c7 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ htmlcov/ .coverage .coverage.* .cache +.env nosetests.xml coverage.xml *,cover diff --git a/CHANGELOG.md b/CHANGELOG.md index 3daf146a73d..38c50769353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Under the hood - If column config says quote, use quoting in SQL for adding a comment. ([#2539](https://github.com/fishtown-analytics/dbt/issues/2539), [#2733](https://github.com/fishtown-analytics/dbt/pull/2733)) +- Added support for running docker-based tests under Linux. ([#2739](https://github.com/fishtown-analytics/dbt/issues/2739)) ### Features - Specify all three logging levels (`INFO`, `WARNING`, `ERROR`) in result logs for commands `test`, `seed`, `run`, `snapshot` and `source snapshot-freshness` ([#2680](https://github.com/fishtown-analytics/dbt/pull/2680), [#2723](https://github.com/fishtown-analytics/dbt/pull/2723)) @@ -12,6 +13,7 @@ Contributors: - [@tpilewicz](https://github.com/tpilewicz) ([#2723](https://github.com/fishtown-analytics/dbt/pull/2723)) +- [@heisencoder](https://github.com/heisencoder) ([#2739](https://github.com/fishtown-analytics/dbt/issues/2739)) ## dbt 0.18.0 (September 03, 2020) diff --git a/Dockerfile b/Dockerfile index 372fe145434..80fa10d5d40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ FROM ubuntu:18.04 ENV DEBIAN_FRONTEND noninteractive -ARG DOCKERIZE_VERSION=v0.6.1 RUN apt-get update && \ apt-get dist-upgrade -y && \ @@ -19,20 +18,33 @@ RUN apt-get update && \ python3.9 python3.9-dev python3.9-venv && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN useradd -mU dbt_test_user -RUN mkdir /usr/app && chown dbt_test_user /usr/app -RUN mkdir /home/tox && chown dbt_test_user /home/tox +ARG DOCKERIZE_VERSION=v0.6.1 RUN curl -LO https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -WORKDIR /usr/app -VOLUME /usr/app - RUN pip3 install -U "tox==3.14.4" wheel "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools # tox fails if the 'python' interpreter (python2) doesn't have `tox` installed RUN pip install -U "tox==3.14.4" "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools +# These args are passed in via docker-compose, which reads then from the .env file. +# On Linux, run `make .env` to create the .env file for the current user. +# On MacOS and Windows, these can stay unset. +ARG USER_ID +ARG GROUP_ID + +RUN if [ ${USER_ID:-0} -ne 0 ] && [ ${GROUP_ID:-0} -ne 0 ]; then \ + groupadd -g ${GROUP_ID} dbt_test_user && \ + useradd -m -l -u ${USER_ID} -g ${GROUP_ID} dbt_test_user; \ + else \ + useradd -mU -l dbt_test_user; \ + fi +RUN mkdir /usr/app && chown dbt_test_user /usr/app +RUN mkdir /home/tox && chown dbt_test_user /home/tox + +WORKDIR /usr/app +VOLUME /usr/app + USER dbt_test_user ENV PYTHONIOENCODING=utf-8 diff --git a/Makefile b/Makefile index 5da89e89584..88593741cac 100644 --- a/Makefile +++ b/Makefile @@ -5,25 +5,33 @@ changed_tests := `git status --porcelain | grep '^\(M\| M\|A\| A\)' | awk '{ pri install: pip install -e . -test: +test: .env @echo "Full test run starting..." @time docker-compose run test tox -test-unit: +test-unit: .env @echo "Unit test run starting..." @time docker-compose run test tox -e unit-py36,flake8 -test-integration: +test-integration: .env @echo "Integration test run starting..." @time docker-compose run test tox -e integration-postgres-py36,integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36 -test-quick: +test-quick: .env @echo "Integration test run starting..." @time docker-compose run test tox -e integration-postgres-py36 -- -x +# This rule creates a file named .env that is used by docker-compose for passing +# the USER_ID and GROUP_ID arguments to the Docker image. +.env: + @echo USER_ID=$(shell id -u) > .env + @echo GROUP_ID=$(shell id -g) >> .env + @time docker-compose build + clean: rm -f .coverage rm -rf .eggs/ + rm -f .env rm -rf .tox/ rm -rf build/ rm -rf dbt.egg-info/ diff --git a/docker-compose.yml b/docker-compose.yml index 93ac2c8cc7b..220b31edbd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,10 @@ services: build: context: . dockerfile: Dockerfile + args: + # Run `make .env` to set $USER_ID and $GROUP_ID + USER_ID: ${USER_ID} + GROUP_ID: ${GROUP_ID} command: "/root/.virtualenvs/dbt/bin/pytest" env_file: - ./test.env