From 28be3f3e539b6f39c8c9d02d16ee8a3c9460a9dd Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Wed, 2 Sep 2020 10:02:07 -0700 Subject: [PATCH 1/4] Fixes xenial docker build We need a recent version of dh-virtualenv. As for Focal, for Xenial we now pull from the Debian [sic] unstable [sic] repos to get a recent version. In Focal, we could reuse the already-available debian-archive-keyring to verify integrity, but in Xenial, the debian-archive-keyring is too old, Jessie-era (2014-11 vs 2018-09, judging by the timestamps). So for Xenial, we must fetch the required key over HTTPS and provide that to apt manually. We must also explicitly reference the "unstable" target for dh-virtualenv, since an older version of the package is available in Xenial, and will be preferrred to the unstable version given the cautious apt preferences we configure. --- molecule/builder-xenial/Dockerfile | 17 +++++++++-------- molecule/builder-xenial/image_hash | 4 ++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/molecule/builder-xenial/Dockerfile b/molecule/builder-xenial/Dockerfile index 9c438bb41e..fde58d947a 100644 --- a/molecule/builder-xenial/Dockerfile +++ b/molecule/builder-xenial/Dockerfile @@ -1,5 +1,5 @@ -# ubuntu:xenial-20190122 -FROM ubuntu@sha256:e4a134999bea4abb4a27bc437e6118fdddfb172e1b9d683129b74d254af51675 +# ubuntu:xenial-20200902 +FROM ubuntu@sha256:3dd44f7ca10f07f86add9d0dc611998a1641f501833692a2651c96defe8db940 # additional meta-data makes it easier to clean up, find LABEL org="Freedom of the Press" @@ -7,8 +7,10 @@ LABEL image_name="xenial-sd-builder-app" RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \ apache2-dev \ + apt-transport-https \ aptitude \ coreutils \ + curl \ debhelper \ devscripts \ dh-python \ @@ -31,15 +33,14 @@ RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \ sqlite \ sudo \ tzdata \ - unzip \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + unzip #install dh-virtualenv from debian unstable +RUN curl https://ftp-master.debian.org/keys/archive-key-10.asc -o /tmp/debian-stable-archive-pubkey.asc && \ + apt-key add /tmp/debian-stable-archive-pubkey.asc RUN echo "deb https://deb.debian.org/debian unstable main contrib" > /etc/apt/sources.list.d/debian-unstable.list COPY aptpreferences.conf /etc/apt/preferences.d/debian-unstable -RUN apt-get install -y debian-archive-keyring -RUN ln -s /usr/share/keyrings/debian-archive-keyring.gpg /etc/apt/trusted.gpg.d/ -RUN apt-get update && apt-get install -y dh-virtualenv +RUN apt-get update && apt-get install -y -t unstable dh-virtualenv +RUN apt-get clean && rm -rf /var/lib/apt/lists/* RUN paxctl -cm /usr/bin/python3.5 && mkdir -p /tmp/build diff --git a/molecule/builder-xenial/image_hash b/molecule/builder-xenial/image_hash index 3ae073d024..3b3783c832 100644 --- a/molecule/builder-xenial/image_hash +++ b/molecule/builder-xenial/image_hash @@ -1,2 +1,2 @@ -# sha256 digest quay.io/freedomofpress/sd-docker-builder-xenial:2020_07_27 -9cfdae58bb673fc7c336864faa223732e6ad169a5d1c4ccd5d76217afc5385f6 +# sha256 digest quay.io/freedomofpress/sd-docker-builder-xenial:2020_09_02 +4d726197484c3dadf8510d883f57a86494e9b1b162cee18577198e9e97dd289d From 5a6c3b651722f2495f2f4e495b63b614109fea92 Mon Sep 17 00:00:00 2001 From: John Hensley Date: Wed, 2 Sep 2020 18:03:11 -0400 Subject: [PATCH 2/4] Stop dh-virtualenv pulling in Sid Python 3 packages --- molecule/builder-xenial/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/molecule/builder-xenial/Dockerfile b/molecule/builder-xenial/Dockerfile index fde58d947a..8e1d0d3917 100644 --- a/molecule/builder-xenial/Dockerfile +++ b/molecule/builder-xenial/Dockerfile @@ -33,7 +33,8 @@ RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \ sqlite \ sudo \ tzdata \ - unzip + unzip \ + virtualenv #install dh-virtualenv from debian unstable RUN curl https://ftp-master.debian.org/keys/archive-key-10.asc -o /tmp/debian-stable-archive-pubkey.asc && \ From 424a3bef53c48f178507fe22f744e8127cda373d Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Wed, 2 Sep 2020 15:47:55 -0700 Subject: [PATCH 3/4] Updates xenial builder image hash Fixes to the build logic were just added by @rmol, so I've rebuilt the image and pushed it to facilitate testing by others. --- molecule/builder-xenial/image_hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molecule/builder-xenial/image_hash b/molecule/builder-xenial/image_hash index 3b3783c832..c5c03e9379 100644 --- a/molecule/builder-xenial/image_hash +++ b/molecule/builder-xenial/image_hash @@ -1,2 +1,2 @@ # sha256 digest quay.io/freedomofpress/sd-docker-builder-xenial:2020_09_02 -4d726197484c3dadf8510d883f57a86494e9b1b162cee18577198e9e97dd289d +d62c99bb967e6dd1b276e642a29000656489313c2da2997f6bfbf466509f9b74 From da1eee35d9e423cd2640dba8611cfd29438f7f84 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Wed, 2 Sep 2020 15:27:21 -0700 Subject: [PATCH 4/4] Adds tests to package build logic for py/dhv We now test explicitly for expected versions of: * python3 * dh-virtualenv Since we're pulling from non-Ubuntu repositories. We'll reuse the same test logic in the march toward Focal build support, as well. --- .../tests/test_build_dependencies.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/molecule/builder-xenial/tests/test_build_dependencies.py b/molecule/builder-xenial/tests/test_build_dependencies.py index 6cfce29ba3..b0db89e626 100644 --- a/molecule/builder-xenial/tests/test_build_dependencies.py +++ b/molecule/builder-xenial/tests/test_build_dependencies.py @@ -3,6 +3,8 @@ SECUREDROP_TARGET_PLATFORM = os.environ.get("SECUREDROP_TARGET_PLATFORM") +SECUREDROP_PYTHON_VERSION = os.environ.get("SECUREDROP_PYTHON_VERSION", "3.5") +SECUREDROP_DH_VIRTUALENV_VERSION = os.environ.get("SECUREDROP_DH_VIRTUALENV_VERSION", "1.2.1") testinfra_hosts = [ "docker://{}-sd-app".format(SECUREDROP_TARGET_PLATFORM) ] @@ -38,3 +40,27 @@ def test_build_all_packages_updated(host): c = host.run('aptitude --simulate -y dist-upgrade') assert c.rc == 0 assert "No packages will be installed, upgraded, or removed." in c.stdout + + +def test_python_version(host): + """ + The Python 3 version shouldn't change between LTS releases, but we're + pulling in some packages from Debian for dh-virtualenv support, so + we must be careful not to change Python as well. + """ + c = host.run("python3 --version") + version_string = "Python {}".format(SECUREDROP_PYTHON_VERSION) + assert c.stdout.startswith(version_string) + + +def test_dh_virtualenv(host): + """ + The version of dh-virtualenv in Xenial repos isn't new enough to work + with setuptools >= 50, so we pull it in from Debian. Confirm the expected + recent version of dh-virtualenv is found. Since we're tracking Debian unstable + for this dependency, this check will fail if unstable surprises us with a new + version. + """ + c = host.run("dh_virtualenv --version") + version_string = "dh_virtualenv {}".format(SECUREDROP_DH_VIRTUALENV_VERSION) + assert c.stdout.startswith(version_string)