From 1cd6d82b662bdee77139d92ef7c763c268fcc3eb Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Thu, 21 Nov 2019 13:36:30 -0800 Subject: [PATCH 1/5] Updates dom0 RPM repo URL Migrating away from the S3-bucket hosting for RPMs, and towards a git-lfs-backed solution, same as we're using for the deb packages. Purges the "publish-rpm" logic, since that's no longer necessary. Updated documentation where appropriate. --- .gitignore | 1 - Makefile | 3 -- README.md | 7 +---- dom0/sd-dom0-files.sls | 2 +- scripts/publish-rpm | 65 ------------------------------------------ 5 files changed, 2 insertions(+), 76 deletions(-) delete mode 100755 scripts/publish-rpm diff --git a/.gitignore b/.gitignore index 5bbd3a2a..c688d8f0 100644 --- a/.gitignore +++ b/.gitignore @@ -113,7 +113,6 @@ builder/packages/securedrop-workstation-grsec/debian/files # rpm package build artifacts *.rpm -rpm-repo/ *.tar.gz rpm-build/BUILD/ rpm-build/BUILDROOT/ diff --git a/Makefile b/Makefile index 45e98a93..a8bc6d9c 100644 --- a/Makefile +++ b/Makefile @@ -131,9 +131,6 @@ flake8: ## Lints all Python files with flake8 template: ## Builds securedrop-workstation Qube template RPM ./builder/build-workstation-template -publish-rpm: ## Uploads signed RPMs to dom0 repository - ./scripts/publish-rpm - prep-dom0: prep-salt # Copies dom0 config files for VM updates sudo qubesctl top.enable sd-vm-updates sudo qubesctl top.enable sd-dom0-files diff --git a/README.md b/README.md index 621d8359..64f2040c 100644 --- a/README.md +++ b/README.md @@ -531,12 +531,7 @@ For the Debian packages, see https://github.com/freedomofpress/securedrop-debian For the RPM packages, such as the `securedrop-workstation` TemplateVM package, first build the package (e.g. `make template`), then sign the RPM, as outlined above. -To upload the package to S3, you'll need valid AWS credentials. Talk to a member of the ops team. -Once you have valid credentials configured, install the dependencies (`pip install -r requirements.txt` in a virtualenv), then run: - -``` -./scripts/publish-rpm -``` +To upload the package, submit a PR to https://github.com/freedomofpress/securedrop-workstation-dev-rpm-packages-lfs/ The RPM will immediately be available in dom0. Provided you've run the Salt configurations, find it via: diff --git a/dom0/sd-dom0-files.sls b/dom0/sd-dom0-files.sls index cd3568e3..8abbe319 100644 --- a/dom0/sd-dom0-files.sls +++ b/dom0/sd-dom0-files.sls @@ -38,7 +38,7 @@ dom0-workstation-rpm-repo: gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-securedrop-workstation-test enabled=1 - baseurl=https://dev-bin.ops.securedrop.org/dom0-rpm-repo/ + baseurl=https://yum.securedrop.org/workstation/dom0/f25 name=SecureDrop Workstation Qubes dom0 repo - require: - file: dom0-rpm-test-key diff --git a/scripts/publish-rpm b/scripts/publish-rpm deleted file mode 100755 index 72c4f5bb..00000000 --- a/scripts/publish-rpm +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# Creates a local RPM repo, then pushes its contents to S3, for serving. -set -e -set -u -set -o pipefail - - -REPO_ROOT="$(git rev-parse --show-toplevel)" -RPM_LOCAL_DIR="${REPO_ROOT}/rpm-repo" - - -function list_rpms() { - find "$RPM_LOCAL_DIR" | grep '\.rpm$' -} - -function container_run() { - docker run --rm \ - --network=none \ - -v "$RPM_LOCAL_DIR:/repo" \ - fpf.local/createrepo \ - $@ -} - -# Check that we have local RPMs to build a repo for -if [[ -z "$(list_rpms)" ]]; then - echo "No RPM files found in $RPM_LOCAL_DIR" - echo "Build RPMs and place in that directory, then rerun the publish action." - exit 1 -fi - -# Ensure we have 'aws' installed, otherwise we cannot push to S3 -if ! hash aws > /dev/null 2>&1 ; then - echo "'aws' CLI not found, install requirements" - exit 2 -fi - -# Build container for preparing repo -docker build -t fpf.local/createrepo -f docker/CreateRepoRPM/Dockerfile . - -# TODO: In order to manage state over time, we'll need to *pull* from S3, -# populating the existing local dir with the current state of what's in S3. -# That's a bandwidth-intensive operation, so skipping for now. -# aws --profile sdpackager s3 sync "s3://dev-bin.ops.securedrop.org/dom0-rpm-repo/ ${RPM_LOCAL_DIR}/" - -# Sanity check that we have RPMs locally to upload, and they're already -# signed. -echo "Validating RPM signatures..." -while read -r f; do - fname="$(basename "$f")" - sig_results="$(container_run rpm -Kv "$fname")" - if ! grep -qP '^\s+V4 RSA/SHA(256|512) Signature, key ID \w+: OK$' <<< "$sig_results"; then - echo "Failed to validate signature on $fname" - echo "Is the RPM signed? rpm -Kv showed:" - echo "$sig_results" - exit 3 - fi -done <<< "$(list_rpms)" - -# Use local container for creating repo metadata -container_run createrepo_c . - -# Push created repo dirtree to S3 -aws --profile sdpackager s3 sync \ - --exclude ".empty" \ - "${RPM_LOCAL_DIR}/" s3://dev-bin.ops.securedrop.org/dom0-rpm-repo/ From 8e5cf83627f530a202ccf3fc01890fce17b49097 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Thu, 21 Nov 2019 15:25:12 -0800 Subject: [PATCH 2/5] Removes Python requirements files We were only declaring Python requirements in order to pull in AWS tooling for managing S3 buckets. Since we've migrated to git-lfs for hosting the RPM packages, these requirements are no longer necessary. Hat tip to @creviera & @redshiftzero for the hard work on migrating Pipfile -> requirements.txt so recently. --- README.md | 17 ++------------ requirements.in | 13 ----------- requirements.txt | 59 ------------------------------------------------ 3 files changed, 2 insertions(+), 87 deletions(-) delete mode 100644 requirements.in delete mode 100644 requirements.txt diff --git a/README.md b/README.md index 64f2040c..af7861cb 100644 --- a/README.md +++ b/README.md @@ -238,21 +238,8 @@ qvm-create --template securedrop-workstation test-securedrop-workstation --class ### Building workstation deb packages -``` -# go to the builder/ directory: -cd builder/packages -# build a specific package (e.g, grsecurity metapackage) -make securedrop-workstation-grsec -# OR build all the packages -make all -# run the tests -virtualenv --python python3 .venv -source .venv/bin/activate -pip install -r requirements.txt -# install test requirements and run the test -apt install lintian -make test -``` +Debian packages for the SecureDrop Workstation components are maintained in a separate repository: +https://github.com/freedomofpress/securedrop-debian-packaging/ ### Building workstation rpm packages diff --git a/requirements.in b/requirements.in deleted file mode 100644 index 08f47157..00000000 --- a/requirements.in +++ /dev/null @@ -1,13 +0,0 @@ -awscli==1.16.270 -botocore==1.13.6 -colorama==0.3.9 -docutils==0.14 -jmespath==0.9.4 -pip-tools==4.2.0 -pyasn1==0.4.5 -python-dateutil==2.8.0 -pyyaml==5.1 -rsa==3.4.2 -s3transfer==0.2.1 -six==1.12.0 -urllib3==1.25.3 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 249c4085..00000000 --- a/requirements.txt +++ /dev/null @@ -1,59 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --generate-hashes --output-file=requirements.txt requirements.in -# -awscli==1.16.270 \ - --hash=sha256:3e2faac56f4fc7ce23a4a0625f422b908b25cb1c178e5f8879ddfc4e80a57549 \ - --hash=sha256:5b37212405ce6c05262abc605fbf7099bea5f47b3192f236375a7f21d2bbe9ef -botocore==1.13.6 \ - --hash=sha256:ac1a5caa10e3c4452714b17e6f30f05b4b6e57e0c80b19c1f4d72b234edf6646 \ - --hash=sha256:fa6b9e619423f3891e7c11b98f2183da8173e3fed995271e93fd4a712ef45777 -click==7.0 \ - --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ - --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ - # via pip-tools -colorama==0.3.9 \ - --hash=sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda \ - --hash=sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1 -docutils==0.14 \ - --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \ - --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \ - --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 -jmespath==0.9.4 \ - --hash=sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6 \ - --hash=sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c -pip-tools==4.2.0 \ - --hash=sha256:123174aabf7f4a63dd6e0bfc8aeeb5eaddbecb75a41e9f0dd4c447b1f2de14f7 \ - --hash=sha256:5427ea4dcc175649723985fbcace9b2d8f46f9adbcc63bc2d7b247d9bcc74917 -pyasn1==0.4.5 \ - --hash=sha256:da2420fe13a9452d8ae97a0e478adde1dee153b11ba832a95b223a2ba01c10f7 \ - --hash=sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e -python-dateutil==2.8.0 \ - --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \ - --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e -pyyaml==5.1 \ - --hash=sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c \ - --hash=sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95 \ - --hash=sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2 \ - --hash=sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4 \ - --hash=sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad \ - --hash=sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba \ - --hash=sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1 \ - --hash=sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e \ - --hash=sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673 \ - --hash=sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13 \ - --hash=sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19 -rsa==3.4.2 \ - --hash=sha256:25df4e10c263fb88b5ace923dd84bf9aa7f5019687b5e55382ffcdb8bede9db5 \ - --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd -s3transfer==0.2.1 \ - --hash=sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d \ - --hash=sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba -six==1.12.0 \ - --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ - --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 -urllib3==1.25.3 \ - --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \ - --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232 From 8a9ffa55e300c5eff55ceb36c6143f189c58a00c Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Thu, 21 Nov 2019 16:09:21 -0800 Subject: [PATCH 3/5] Default to sd-dev in make clone action Closes #334. --- scripts/clone-to-dom0 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/clone-to-dom0 b/scripts/clone-to-dom0 index 29e2118e..963ea711 100755 --- a/scripts/clone-to-dom0 +++ b/scripts/clone-to-dom0 @@ -14,7 +14,7 @@ if [[ "$(hostname)" != "dom0" ]]; then fi # Support environment variable overrides, but provide sane defaults. -dev_vm="${SECUREDROP_DEV_VM:-work}" +dev_vm="${SECUREDROP_DEV_VM:-sd-dev}" dev_dir="${SECUREDROP_DEV_DIR:-/home/user/securedrop-workstation}" # The dest directory in dom0 is not customizable. From 39c66a978fcf1070c30cab5b860c5faae9084e6c Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Thu, 21 Nov 2019 16:10:32 -0800 Subject: [PATCH 4/5] Refines sys-firewall pubkey rc.local config The line was being appended every time, rather than being enforced as present once. Closes #347. --- dom0/sd-sys-firewall-files.sls | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dom0/sd-sys-firewall-files.sls b/dom0/sd-sys-firewall-files.sls index 11dbaf23..e75d3083 100644 --- a/dom0/sd-sys-firewall-files.sls +++ b/dom0/sd-sys-firewall-files.sls @@ -15,12 +15,9 @@ sys-firewall-rpm-test-key-script: - mode: 755 sys-firewall-rpm-test-key-rclocal: - file.line: + file.append: - name: /rw/config/rc.local - - mode: insert - - location: end - - match: "/rw/config/sd-copy-rpm-repo-pubkey.sh" - - content: "/rw/config/sd-copy-rpm-repo-pubkey.sh" + - text: "/rw/config/sd-copy-rpm-repo-pubkey.sh" sys-firewall-rpm-test-key-import: cmd.run: From 79838488e0ce0f4fdc41759417c5ff0d06828b0e Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Fri, 22 Nov 2019 10:28:22 -0800 Subject: [PATCH 5/5] Uses "test" URL for Qubes dom0 repo We'll reserve the non-test URLs for stable versions. For now, we're still slinging nightlies around, so let's be clear about expectations. --- dom0/sd-dom0-files.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dom0/sd-dom0-files.sls b/dom0/sd-dom0-files.sls index 8abbe319..d82829a1 100644 --- a/dom0/sd-dom0-files.sls +++ b/dom0/sd-dom0-files.sls @@ -38,7 +38,7 @@ dom0-workstation-rpm-repo: gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-securedrop-workstation-test enabled=1 - baseurl=https://yum.securedrop.org/workstation/dom0/f25 + baseurl=https://yum-test.securedrop.org/workstation/dom0/f25 name=SecureDrop Workstation Qubes dom0 repo - require: - file: dom0-rpm-test-key