diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f350d7d665..4d61769f15a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ common-steps: - &restorecache restore_cache: - key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }} + key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }} paths: - /caches/layers.tar.gz @@ -35,63 +35,21 @@ common-steps: command: | set +o pipefail docker images - fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}') - cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" ./bin/dev-shell true + fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') + cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" ./bin/dev-shell true - &saveimagelayers run: name: Save Docker image layer cache command: | docker images - docker save -o /caches/layers.tar securedrop-test-xenial-py3:latest + docker save -o /caches/layers.tar securedrop-test-focal-py3:latest - &savecache - save_cache: - key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }} - paths: - - /caches/layers.tar - - - &focalcreatecachedir - run: - name: Ensure cache dir exists and permissions are good - command: | - sudo mkdir -p /focalcaches && sudo chown circleci: -R /focalcaches - - - &focalrestorecache - restore_cache: - key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }} - paths: - - /focalcaches/layers.tar.gz - - - &focalloadimagelayers - run: - name: Load image layer cache on Focal - command: | - set +o pipefail - docker load -i /focalcaches/layers.tar || true - - - &focaldockerimagebuild - run: - name: Build Docker images for Focal - command: | - set +o pipefail - docker images - fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') - cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" BASE_OS=focal ./bin/dev-shell true - - - &focalsaveimagelayers - run: - name: Save Docker image layer cache on Focal - command: | - docker images - docker save -o /focalcaches/layers.tar securedrop-test-focal-py3:latest - - - &focalsavecache save_cache: key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }} paths: - - /focalcaches/layers.tar - + - /caches/layers.tar version: 2 jobs: @@ -123,47 +81,13 @@ jobs: name: Run shellcheck command: make shellcheck - focal-app-tests: - machine: - image: ubuntu-2004:202010-01 - enabled: true - environment: - DOCKER_API_VERSION: 1.23 - BASE_OS: focal - parallelism: 3 - steps: - - checkout - - *rebaseontarget - - *focalcreatecachedir - - *focalrestorecache - - *focalloadimagelayers - - *focaldockerimagebuild - - *focalsaveimagelayers - - *focalsavecache - - - run: - name: Run tests on Focal - command: | - BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|update-builder)") - if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi - export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo) - fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') - DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make test-focal - no_output_timeout: 15m - - - store_test_results: - path: ~/project/test-results-focal - - - store_artifacts: - path: ~/project/test-results-focal - app-tests: machine: image: ubuntu-2004:202010-01 enabled: true environment: DOCKER_API_VERSION: 1.23 - BASE_OS: xenial + BASE_OS: focal parallelism: 3 steps: - checkout @@ -181,8 +105,8 @@ jobs: BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|update-builder)") if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo) - fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}') - DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make test + fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') + DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make test no_output_timeout: 15m - store_test_results: @@ -197,7 +121,7 @@ jobs: enabled: true environment: DOCKER_API_VERSION: 1.23 - BASE_OS: xenial + BASE_OS: focal parallelism: 20 steps: - checkout @@ -216,8 +140,8 @@ jobs: if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi sudo apt update && sudo apt install python3-sh python3-babel export LOCALES="$(/usr/bin/python3 securedrop/i18n_tool.py list-locales --lines | circleci tests split | tr '\n' ' ')" - fromtag=$(docker images | grep securedrop-test-xenial-py3 | head -n1 | awk '{print $2}') - DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make translation-test + fromtag=$(docker images | grep securedrop-test-focal-py3 | head -n1 | awk '{print $2}') + DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make translation-test - store_test_results: path: ~/project/test-results @@ -284,7 +208,7 @@ jobs: enabled: true environment: DOCKER_API_VERSION: 1.23 - BASE_OS: xenial + BASE_OS: focal steps: - checkout - *rebaseontarget @@ -298,14 +222,14 @@ jobs: - run: name: Check Python dependencies for CVEs command: | - fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}') - DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade safety && make -C .. safety" + fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') + DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade safety && make -C .. safety" - run: name: Run static security testing on source code command: | - fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}') - DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade bandit && make -C .. bandit" + fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') + DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade bandit && make -C .. bandit" staging-test-with-rebase: @@ -340,38 +264,6 @@ jobs: - store_artifacts: path: ~/sd/junit - staging-test-with-rebase-focal: - machine: - image: ubuntu-2004:202010-01 - enabled: true - - working_directory: ~/sd - steps: - - checkout - - *rebaseontarget - - *installenchant - - - run: - name: Run Staging tests on GCE - command: | - BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n)") - if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi - BASE_OS=focal make ci-go - no_output_timeout: 35m - - - run: - name: Ensure environment torn down - # Always report true, since env should will destroyed already - # if all tests passed. - command: make ci-teardown || true - when: always - - - store_test_results: - path: ~/sd/junit - - - store_artifacts: - path: ~/sd/junit - deb-tests: docker: - image: cimg/python:3.7 @@ -389,21 +281,6 @@ jobs: if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi make ci-deb-tests - deb-tests-focal: - docker: - - image: cimg/python:3.7 - environment: - LC_ALL: C.UTF-8 - LANG: C.UTF-8 - steps: - - run: sudo apt-get update && sudo apt-get install -y make virtualenv enchant jq python3-dev build-essential rsync - - checkout - - setup_remote_docker - - run: - name: Test Debian package build on Focal - command: | - make ci-deb-tests-focal - workflows: version: 2 securedrop_ci: @@ -417,14 +294,6 @@ workflows: - /update-builder-.*/ requires: - lint - - focal-app-tests: - filters: - branches: - ignore: - - /i18n-.*/ - - /update-builder-.*/ - requires: - - lint - admin-tests: filters: branches: @@ -451,13 +320,6 @@ workflows: - /i18n-.*/ requires: - lint - - staging-test-with-rebase-focal: - filters: - branches: - ignore: - - /i18n-.*/ - requires: - - lint - translation-tests: requires: - lint @@ -468,13 +330,6 @@ workflows: - /update-builder-.*/ requires: - lint - - deb-tests-focal: - filters: - branches: - ignore: - - /i18n-.*/ - requires: - - lint nightly: triggers: @@ -497,6 +352,5 @@ workflows: - develop jobs: - deb-tests - - deb-tests-focal - translation-tests - fetch-tor-debs diff --git a/Makefile b/Makefile index e912742755e..56a642ad12a 100644 --- a/Makefile +++ b/Makefile @@ -219,10 +219,7 @@ test: ## Run the test suite in a Docker container. @echo .PHONY: test-focal -test-focal: ## Run the test suite in a Docker container. - @echo "███ Running SecureDrop application tests..." - @BASE_OS='focal' $(DEVSHELL) $(SDBIN)/run-test -v $${TESTFILES:-tests} - @echo +test-focal: test .PHONY: docker-vnc docker-vnc: ## Open a VNC connection to a running Docker instance. @@ -230,7 +227,6 @@ docker-vnc: ## Open a VNC connection to a running Docker instance. @$(SDROOT)/devops/scripts/vnc-docker-connect.sh @echo -# Xenial upgrade targets .PHONY: upgrade-start upgrade-start: ## Boot an upgrade test environment using libvirt. @echo "███ Starting upgrade test environment..." @@ -344,14 +340,6 @@ ci-deb-tests: ## Test SecureDrop Debian packages in CI environment. @$(SDROOT)/devops/scripts/test-built-packages.sh @echo -.PHONY: ci-deb-tests-focal -ci-deb-tests-focal: ## Test SecureDrop Debian packages in CI environment. - @echo "███ Running Debian package tests in CI..." - @$(SDROOT)/devops/scripts/test-built-packages.sh focal - @echo - - - .PHONY: build-gcloud-docker build-gcloud-docker: ## Build Docker container for Google Cloud SDK. @echo "Building Docker container for Google Cloud SDK..." @@ -368,7 +356,7 @@ vagrant-package: ## Package a Vagrant box of the last stable SecureDrop release @echo .PHONY: fetch-tor-packages -fetch-tor-packages: ## Retrieves the most recent Tor packages for Xenial, for apt repo. +fetch-tor-packages: ## Retrieves the most recent Tor packages, for apt repo. @echo "Fetching most recent Tor packages..." @$(SDROOT)/devops/scripts/fetch-tor-packages.sh @echo diff --git a/devops/gce-nested/ci-go.sh b/devops/gce-nested/ci-go.sh index 850324eccda..ff80aa107ef 100755 --- a/devops/gce-nested/ci-go.sh +++ b/devops/gce-nested/ci-go.sh @@ -12,7 +12,7 @@ set -e set -u set -o pipefail -export BASE_OS="${BASE_OS:-xenial}" +export BASE_OS="${BASE_OS:-focal}" ./devops/gce-nested/gce-start.sh ./devops/gce-nested/gce-runner.sh diff --git a/devops/gce-nested/gce-runner.sh b/devops/gce-nested/gce-runner.sh index 374f0493f19..955e9372960 100755 --- a/devops/gce-nested/gce-runner.sh +++ b/devops/gce-nested/gce-runner.sh @@ -4,7 +4,7 @@ # for storage as artifacts on the build, so devs can review via web. set -e set -u -BASE_OS="${BASE_OS:-xenial}" +BASE_OS="${BASE_OS:-focal}" TOPLEVEL="$(git rev-parse --show-toplevel)" @@ -56,11 +56,6 @@ copy_securedrop_repo # The test results should be collected regardless of pass/fail, # so register a trap to ensure the fetch always runs. trap fetch_junit_test_results EXIT -if [ "${BASE_OS:-'xenial'}" = "xenial" ] -then - ssh_gce "make build-debs-notest" - ssh_gce "make staging" -else - ssh_gce "make build-debs-notest-focal" - ssh_gce "make staging-focal" -fi + +ssh_gce "make build-debs-notest" +ssh_gce "make staging" diff --git a/devops/scripts/select-staging-env b/devops/scripts/select-staging-env index df67de7c6ac..b11ecb514fd 100755 --- a/devops/scripts/select-staging-env +++ b/devops/scripts/select-staging-env @@ -13,14 +13,13 @@ set -o pipefail # Support overrides for LTS version -securedrop_platform_suffix="-${1:-xenial}" +securedrop_platform_suffix="-${1:-focal}" # Respect explicit choice of Vagrant provider if set. if [[ -n "${VAGRANT_DEFAULT_PROVIDER:-}" ]] ; then securedrop_vm_provider="${VAGRANT_DEFAULT_PROVIDER}" # Check whether it appears we're running in Qubes, in which case the standard Vagrant -# logic will not work at all, due to lack of nested virt support. The Qubes staging -# environment uses Xenial template VMs only, so we also suppress the platform suffix. +# logic will not work at all, due to lack of nested virt support. elif printenv | grep -q ^QUBES_ ; then securedrop_vm_provider="qubes" elif [[ "${OSTYPE:-}" == "linux-gnu" ]]; then diff --git a/devops/scripts/test-built-packages.sh b/devops/scripts/test-built-packages.sh index 79af577b1e0..70eaba326ee 100755 --- a/devops/scripts/test-built-packages.sh +++ b/devops/scripts/test-built-packages.sh @@ -5,7 +5,7 @@ set -e set -o pipefail -TARGET_PLATFORM="${1:-xenial}" +TARGET_PLATFORM="${1:-focal}" . ./devops/scripts/boot-strap-venv.sh virtualenv_bootstrap diff --git a/update_version.sh b/update_version.sh index 6109ecfc13c..4a2ef443889 100755 --- a/update_version.sh +++ b/update_version.sh @@ -65,8 +65,6 @@ sed -i "s/\(## ${OLD_VERSION}\)/## ${NEW_VERSION}\n\n\n\n\1/g" changelog.md export DEBEMAIL="${DEBEMAIL:-securedrop@freedom.press}" export DEBFULLNAME="${DEBFULLNAME:-SecureDrop Team}" -# Update the Xenial changelog in the Debian package -dch -b -v "${NEW_VERSION}+xenial" -D xenial -c install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/changelog-xenial # Update the Focal changelog in the Debian package dch -b -v "${NEW_VERSION}+focal" -D focal -c install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/changelog-focal # Commit the change