diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ede3fadb9..1b5425fbff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,11 +12,12 @@ jobs: - run: make ci-lint - tests: + trusty-app-tests: machine: enabled: true environment: - DOCKER_API_VERSION=1.23 + DOCKER_API_VERSION: 1.23 + BASE_OS: trusty parallelism: 3 steps: - checkout @@ -30,7 +31,73 @@ jobs: sudo mkdir -p /caches && sudo chown circleci: -R /caches - restore_cache: - key: v1-sd-layers-{{ checksum "securedrop/Dockerfile" }} + key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/trusty/Dockerfile" }} + paths: + - /caches/layers.tar.gz + + - run: + name: Load image layer cache + command: | + set +o pipefail + docker load -i /caches/layers.tar |true + + - run: + name: Build Docker image + command: | + set +o pipefail + docker images + fromtag=$(docker images |grep securedrop-test-trusty |head -n1 |awk '{print $2}') + cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-trusty:${fromtag:-latest}" ./bin/dev-shell true + + - run: + name: Save Docker image layer cache + command: | + docker images + docker save -o /caches/layers.tar securedrop-test-trusty:latest + + - save_cache: + key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/trusty/Dockerfile" }} + paths: + - /caches/layers.tar + + - run: + name: Make test results directory + command: mkdir -p ~/test-results + + - run: + name: Run tests + command: | + export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo) + docker rm -f securedrop-test-trusty || true + fromtag=$(docker images |grep securedrop-test-trusty |head -n1 |awk '{print $2}') + cd securedrop && DOCKER_RUN_ARGUMENTS=$(bash <(curl -s https://codecov.io/env)) DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-trusty:${fromtag:-latest}" make test + + - store_test_results: + path: ~/test-results + + - store_artifacts: + path: ~/test-results + + xenial-app-tests: + machine: + enabled: true + environment: + DOCKER_API_VERSION: 1.23 + BASE_OS: xenial + parallelism: 3 + steps: + - checkout + - run: + name: Rebase on-top of github target + command: ./devops/scripts/rebase-ci.sh + + - run: + name: Ensure cache dir exists and permissions are good + command: | + sudo mkdir -p /caches && sudo chown circleci: -R /caches + + - restore_cache: + key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/Dockerfile" }} paths: - /caches/layers.tar.gz @@ -45,29 +112,31 @@ jobs: command: | set +o pipefail docker images - fromtag=$(docker images |grep securedrop-test |head -n1 |awk '{print $2}') - cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test:${fromtag:-latest}" ./bin/dev-shell true + fromtag=$(docker images |grep securedrop-test-xenial |head -n1 |awk '{print $2}') + cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial:${fromtag:-latest}" ./bin/dev-shell true - run: name: Save Docker image layer cache command: | docker images - docker save -o /caches/layers.tar securedrop-test:latest + docker save -o /caches/layers.tar securedrop-test-xenial:latest - save_cache: - key: v1-sd-layers-{{ checksum "securedrop/Dockerfile" }} + key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/Dockerfile" }} paths: - /caches/layers.tar - - run: mkdir -p ~/test-results + - run: + name: Make test results directory + command: mkdir -p ~/test-results - run: name: Run tests command: | export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo) - docker rm -f securedrop-test || true - fromtag=$(docker images |grep securedrop-test |head -n1 |awk '{print $2}') - cd securedrop && DOCKER_RUN_ARGUMENTS=$(bash <(curl -s https://codecov.io/env)) DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test:${fromtag:-latest}" make test + docker rm -f securedrop-test-xenial || true + fromtag=$(docker images |grep securedrop-test-xenial |head -n1 |awk '{print $2}') + cd securedrop && DOCKER_RUN_ARGUMENTS=$(bash <(curl -s https://codecov.io/env)) DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial:${fromtag:-latest}" make test - store_test_results: path: ~/test-results @@ -159,11 +228,11 @@ workflows: securedrop_ci: jobs: - lint - - tests + - trusty-app-tests + - xenial-app-tests - admin-tests - updater-gui-tests - static-analysis-and-no-known-cves - staging-test-with-rebase: requires: - lint - - updater-gui-tests diff --git a/securedrop/Makefile b/securedrop/Makefile index 901f2dfd9e..8dec0c8a36 100644 --- a/securedrop/Makefile +++ b/securedrop/Makefile @@ -17,9 +17,13 @@ lint-full: ## Run the python linter with nothing disabled find . -name '*.py' | xargs pylint .PHONY: test -test: ## Run the test suite in a dockerized environment +test: ## Run the test suite in a Ubuntu 14.04 (Trusty) dockerized environment ./bin/dev-shell ./bin/run-test -v $${TESTFILES:-tests} +.PHONY: test-xenial +test-xenial: ## Run the test suite in a Ubuntu 16.04 (Xenial) dockerized environment + BASE_OS=xenial ./bin/dev-shell ./bin/run-test -v $${TESTFILES:-tests} + .PHONY: dev dev: ## Run the dev server DOCKER_RUN_ARGUMENTS='-p127.0.0.1:8080:8080 -p127.0.0.1:8081:8081 -p127.0.0.1:5901:5901' ./bin/dev-shell ./bin/run diff --git a/securedrop/bin/dev-shell b/securedrop/bin/dev-shell index 8c47e2bf99..eb4b90d420 100755 --- a/securedrop/bin/dev-shell +++ b/securedrop/bin/dev-shell @@ -9,28 +9,48 @@ set -eu TOPLEVEL=$(git rev-parse --show-toplevel) source "${BASH_SOURCE%/*}/../../devops/scripts/ticker" +if ! test -n "${BASE_OS:-}" ; then + # If no base OS was specified, then we use Trusty + BASE_OS=trusty +fi + +function exit_if_not_supported_base_image() { + # Currently we only support Xenial or Trusty. + if [[ "$1" != "xenial" && "$1" != "trusty" ]] + then + echo "BASE_OS must be trusty or xenial" + exit 1 + fi +} + function docker_image() { + exit_if_not_supported_base_image $1 + docker build \ ${DOCKER_BUILD_ARGUMENTS:-} \ --build-arg=USER_ID="$(id -u)" \ --build-arg=USER_NAME="${USER:-root}" \ - -t securedrop-test "${TOPLEVEL}/securedrop" + -t "securedrop-test-${1}" \ + --file "${TOPLEVEL}/securedrop/dockerfiles/${1}/Dockerfile" \ + "${TOPLEVEL}/securedrop" } function docker_run() { + exit_if_not_supported_base_image $1 + find . \( -name '*.pyc' -o -name __pycache__ \) -delete docker run \ --rm \ --user "${USER:-root}" \ --volume "${TOPLEVEL}:${TOPLEVEL}" \ --workdir "${TOPLEVEL}/securedrop" \ - -ti ${DOCKER_RUN_ARGUMENTS:-} securedrop-test "$@" + -ti ${DOCKER_RUN_ARGUMENTS:-} "securedrop-test-${1}" "${@:2}" } if test -n "${CIRCLE_SHA1:-}" ; then - docker_image + docker_image $BASE_OS else - ticker docker_image + ticker docker_image $BASE_OS fi -docker_run "$@" +docker_run $BASE_OS "$@" diff --git a/securedrop/Dockerfile b/securedrop/dockerfiles/trusty/Dockerfile similarity index 100% rename from securedrop/Dockerfile rename to securedrop/dockerfiles/trusty/Dockerfile diff --git a/securedrop/dockerfiles/xenial/Dockerfile b/securedrop/dockerfiles/xenial/Dockerfile new file mode 100644 index 0000000000..e64c65b165 --- /dev/null +++ b/securedrop/dockerfiles/xenial/Dockerfile @@ -0,0 +1,31 @@ +# ubuntu 16.04 image - 01-2019 +FROM ubuntu@sha256:b967b9f2a5625231a22db642609e61b7b1a5481128f51fe771e91bb92e0a35d0 +ARG USER_NAME +ENV USER_NAME ${USER_NAME:-root} +ARG USER_ID +ENV USER_ID ${USER_ID:-0} + +RUN apt-get update && \ + apt-get install -y devscripts \ + python-pip libpython2.7-dev libssl-dev secure-delete \ + gnupg2 ruby redis-server firefox git xvfb haveged curl \ + gettext paxctl x11vnc enchant libffi-dev sqlite3 gettext sudo + +ENV FIREFOX_CHECKSUM=179647c2d47e2acf2cf3596f21c8cbea8eda4b271d4d09d7b18be55553751912 +RUN curl -LO https://launchpad.net/~ubuntu-mozilla-security/+archive/ubuntu/ppa/+build/11952510/+files/firefox-locale-en_51.0.1+build2-0ubuntu0.16.04.2_amd64.deb && \ + shasum -a 256 firefox*deb && \ + echo "${FIREFOX_CHECKSUM} firefox-locale-en_51.0.1+build2-0ubuntu0.16.04.2_amd64.deb" | shasum -a 256 -c - && \ + dpkg -i firefox*deb && apt-get install -f && \ + paxctl -cm /usr/lib/firefox/firefox + +RUN gem install sass -v 3.4.23 + +COPY requirements requirements +RUN pip install -r requirements/securedrop-app-code-requirements.txt && \ + pip install -r requirements/test-requirements.txt + +RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi + +STOPSIGNAL SIGKILL + +EXPOSE 8080 8081 5901 diff --git a/securedrop/requirements/test-requirements.in b/securedrop/requirements/test-requirements.in index 60eca542d3..e8768e4bf2 100644 --- a/securedrop/requirements/test-requirements.in +++ b/securedrop/requirements/test-requirements.in @@ -7,4 +7,5 @@ py pytest pytest-cov pytest-mock +requests selenium < 3 diff --git a/securedrop/requirements/test-requirements.txt b/securedrop/requirements/test-requirements.txt index a875f109f4..05d90f9f20 100644 --- a/securedrop/requirements/test-requirements.txt +++ b/securedrop/requirements/test-requirements.txt @@ -7,12 +7,15 @@ attrs==17.4.0 # via pytest beautifulsoup4==4.6.0 blinker==1.4 +certifi==2018.11.29 # via requests +chardet==3.0.4 # via requests click==6.7 # via flask, pip-tools coverage==4.4.2 # via pytest-cov first==2.0.1 # via pip-tools flask-testing==0.7.1 flask==1.0.2 # via flask-testing funcsigs==1.0.2 # via mock, pytest +idna==2.8 # via requests itsdangerous==0.24 # via flask jinja2==2.10 # via flask markupsafe==1.0 # via jinja2 @@ -24,6 +27,8 @@ py==1.5.2 pytest-cov==2.5.1 pytest-mock==1.7.1 pytest==3.3.2 +requests==2.21.0 selenium==2.53.6 six==1.11.0 # via mock, pip-tools, pytest +urllib3==1.24.1 # via requests werkzeug==0.14.1 # via flask