From f4eac01ee21250f15ead22a0253b24f9f872f785 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Mon, 25 Nov 2019 10:09:18 -0800 Subject: [PATCH 1/6] Updates config tests for dom0 repo URL Should have been included as part of #349. That's what I get for omitting `make test` as part of the test plan. =) --- tests/test_dom0_rpm_repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dom0_rpm_repo.py b/tests/test_dom0_rpm_repo.py index fa767e03..91f02ec9 100644 --- a/tests/test_dom0_rpm_repo.py +++ b/tests/test_dom0_rpm_repo.py @@ -26,7 +26,7 @@ def test_rpm_repo_config(self): "gpgcheck=1", "gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-securedrop-workstation-test", # noqa "enabled=1", - "baseurl=https://dev-bin.ops.securedrop.org/dom0-rpm-repo/", + "baseurl=https://yum-test.securedrop.org/workstation/dom0/f25", "name=SecureDrop Workstation Qubes dom0 repo", ] with open(repo_file, "r") as f: From 0f11ac3d5523095f732f2a2e2fdf676ca47b8ffc Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Thu, 21 Nov 2019 16:42:54 -0800 Subject: [PATCH 2/6] Consolidates all Salt top files There's no need for a 1:1 correspondence between Salt states and top files. A single top-level provides the same capability to map Salt states against target VMs. When running, Salt will merge all separate top files to plan the execution steps anyway. Let's consolidate to make the config flatter and more readable. --- Makefile | 42 +++++++++----------------- dom0/sd-dom0-files.top | 6 ---- dom0/sd-dom0-qvm-rpc.top | 6 ---- dom0/sd-export-files.top | 6 ---- dom0/sd-export.top | 6 ---- dom0/sd-gpg-files.top | 6 ---- dom0/sd-gpg.top | 6 ---- dom0/sd-proxy-files.top | 7 ----- dom0/sd-proxy.top | 6 ---- dom0/sd-svs-config.top | 3 -- dom0/sd-svs-disp-files.top | 7 ----- dom0/sd-svs-disp.top | 6 ---- dom0/sd-svs-files.top | 6 ---- dom0/sd-svs.top | 6 ---- dom0/sd-sys-firewall-files.top | 6 ---- dom0/sd-sys-vms.top | 6 ---- dom0/sd-vm-updates.top | 9 ------ dom0/sd-whonix-hidserv-key.top | 6 ---- dom0/sd-whonix.top | 6 ---- dom0/sd-workstation-template-files.top | 7 ----- dom0/sd-workstation-template.top | 6 ---- dom0/sd-workstation.top | 39 ++++++++++++++++++++++++ 22 files changed, 53 insertions(+), 151 deletions(-) delete mode 100644 dom0/sd-dom0-files.top delete mode 100644 dom0/sd-dom0-qvm-rpc.top delete mode 100644 dom0/sd-export-files.top delete mode 100644 dom0/sd-export.top delete mode 100644 dom0/sd-gpg-files.top delete mode 100644 dom0/sd-gpg.top delete mode 100644 dom0/sd-proxy-files.top delete mode 100644 dom0/sd-proxy.top delete mode 100644 dom0/sd-svs-config.top delete mode 100644 dom0/sd-svs-disp-files.top delete mode 100644 dom0/sd-svs-disp.top delete mode 100644 dom0/sd-svs-files.top delete mode 100644 dom0/sd-svs.top delete mode 100644 dom0/sd-sys-firewall-files.top delete mode 100644 dom0/sd-sys-vms.top delete mode 100644 dom0/sd-vm-updates.top delete mode 100644 dom0/sd-whonix-hidserv-key.top delete mode 100644 dom0/sd-whonix.top delete mode 100644 dom0/sd-workstation-template-files.top delete mode 100644 dom0/sd-workstation-template.top create mode 100644 dom0/sd-workstation.top diff --git a/Makefile b/Makefile index a8bc6d9c..8d92c6aa 100644 --- a/Makefile +++ b/Makefile @@ -17,49 +17,35 @@ clone: assert-dom0 ## Pulls the latest repo from work VM to dom0 @./scripts/clone-to-dom0 qubes-rpc: prep-salt ## Places default deny qubes-rpc policies for sd-svs and sd-gpg - sudo qubesctl top.enable sd-dom0-qvm-rpc sudo qubesctl --show-output --targets sd-dom0-qvm-rpc state.highstate sd-workstation-template: prep-salt ## Provisions base template for SDW AppVMs - sudo qubesctl top.enable sd-workstation-template - sudo qubesctl top.enable sd-workstation-template-files - sudo qubesctl --show-output --targets sd-workstation-template state.highstate + sudo qubesctl --show-output state.sls sd-workstation-template + sudo qubesctl --show-output --skip-dom0 --targets sd-workstation-template state.highstate sd-proxy: prep-salt ## Provisions SD Proxy VM - sudo qubesctl top.enable sd-proxy - sudo qubesctl top.enable sd-proxy-files - sudo qubesctl --show-output --targets sd-proxy-template state.highstate - sudo qubesctl --show-output --targets sd-proxy state.highstate + sudo qubesctl --show-output state.sls sd-proxy + sudo qubesctl --show-output --skip-dom0 --targets sd-proxy-template state.highstate sd-gpg: prep-salt ## Provisions SD GPG keystore VM - sudo qubesctl top.enable sd-gpg - sudo qubesctl top.enable sd-gpg-files - sudo qubesctl --show-output --targets sd-gpg state.highstate + sudo qubesctl --show-output state.sls sd-gpg + sudo qubesctl --show-output --skip-dom0 --targets sd-gpg state.highstate sd-svs: prep-salt ## Provisions SD SVS VM - sudo qubesctl top.enable sd-svs - sudo qubesctl top.enable sd-svs-files - sudo qubesctl top.enable sd-svs-config - sudo qubesctl --show-output --targets sd-svs-template state.highstate - sudo qubesctl --show-output --targets sd-svs state.highstate + sudo qubesctl --show-output state.sls sd-svs + sudo qubesctl --show-output --skip-dom0 --targets sd-svs-template,sd-svs state.highstate sd-whonix: prep-salt ## Provisions SD Whonix VM - sudo qubesctl top.enable sd-whonix - sudo qubesctl top.enable sd-whonix-hidserv-key - sudo qubesctl --show-output --targets sd-whonix-template state.highstate - sudo qubesctl --show-output --targets sd-whonix state.highstate + sudo qubesctl --show-output state.sls sd-whonix + sudo qubesctl --show-output --skip-dom0 --targets sd-whonix-template,sd-whonix state.highstate sd-svs-disp: prep-salt ## Provisions SD Submission Viewing VM - sudo qubesctl top.enable sd-svs-disp - sudo qubesctl top.enable sd-svs-disp-files - sudo qubesctl --show-output --targets sd-svs-disp-template state.highstate - sudo qubesctl --show-output --targets sd-svs-disp state.highstate + sudo qubesctl --show-output state.sls sd-svs-disp + sudo qubesctl --show-output --skip-dom0 --targets sd-svs-disp-template,sd-svs-disp state.highstate sd-export: prep-salt ## Provisions SD Export VM - sudo qubesctl top.enable sd-export - sudo qubesctl top.enable sd-export-files - sudo qubesctl --show-output --targets sd-export-template state.highstate - sudo qubesctl --show-output --targets sd-export-export-dvm state.highstate + sudo qubesctl --show-output state.sls sd-export + sudo qubesctl --show-output --skip-dom0 --targets sd-export-template,sd-export-usb,sd-export-usb-dvm state.highstate clean-salt: assert-dom0 ## Purges SD Salt configuration from dom0 @echo "Purging Salt config..." diff --git a/dom0/sd-dom0-files.top b/dom0/sd-dom0-files.top deleted file mode 100644 index 3509270d..00000000 --- a/dom0/sd-dom0-files.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-dom0-files diff --git a/dom0/sd-dom0-qvm-rpc.top b/dom0/sd-dom0-qvm-rpc.top deleted file mode 100644 index d283686f..00000000 --- a/dom0/sd-dom0-qvm-rpc.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-dom0-qvm-rpc diff --git a/dom0/sd-export-files.top b/dom0/sd-export-files.top deleted file mode 100644 index 1393d60f..00000000 --- a/dom0/sd-export-files.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - sd-export-template: - - sd-export-files diff --git a/dom0/sd-export.top b/dom0/sd-export.top deleted file mode 100644 index bf1f607b..00000000 --- a/dom0/sd-export.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-export diff --git a/dom0/sd-gpg-files.top b/dom0/sd-gpg-files.top deleted file mode 100644 index 10e6b21d..00000000 --- a/dom0/sd-gpg-files.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - sd-gpg: - - sd-gpg-files diff --git a/dom0/sd-gpg.top b/dom0/sd-gpg.top deleted file mode 100644 index 73d6fa2c..00000000 --- a/dom0/sd-gpg.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-gpg diff --git a/dom0/sd-proxy-files.top b/dom0/sd-proxy-files.top deleted file mode 100644 index c0894486..00000000 --- a/dom0/sd-proxy-files.top +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - sd-proxy-template: - - fpf-apt-test-repo - - sd-proxy-template-files diff --git a/dom0/sd-proxy.top b/dom0/sd-proxy.top deleted file mode 100644 index f4f0a286..00000000 --- a/dom0/sd-proxy.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-proxy diff --git a/dom0/sd-svs-config.top b/dom0/sd-svs-config.top deleted file mode 100644 index f3044991..00000000 --- a/dom0/sd-svs-config.top +++ /dev/null @@ -1,3 +0,0 @@ -base: - sd-svs: - - sd-svs-config diff --git a/dom0/sd-svs-disp-files.top b/dom0/sd-svs-disp-files.top deleted file mode 100644 index 76ecf0f6..00000000 --- a/dom0/sd-svs-disp-files.top +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - sd-svs-disp-template: - - fpf-apt-test-repo - - sd-svs-disp-files diff --git a/dom0/sd-svs-disp.top b/dom0/sd-svs-disp.top deleted file mode 100644 index c6988e7e..00000000 --- a/dom0/sd-svs-disp.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-svs-disp diff --git a/dom0/sd-svs-files.top b/dom0/sd-svs-files.top deleted file mode 100644 index 737b1350..00000000 --- a/dom0/sd-svs-files.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - sd-svs-template: - - sd-svs-files diff --git a/dom0/sd-svs.top b/dom0/sd-svs.top deleted file mode 100644 index 9c5a41f1..00000000 --- a/dom0/sd-svs.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-svs diff --git a/dom0/sd-sys-firewall-files.top b/dom0/sd-sys-firewall-files.top deleted file mode 100644 index 49569369..00000000 --- a/dom0/sd-sys-firewall-files.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - sys-firewall: - - sd-sys-firewall-files diff --git a/dom0/sd-sys-vms.top b/dom0/sd-sys-vms.top deleted file mode 100644 index 10f174d1..00000000 --- a/dom0/sd-sys-vms.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-sys-vms diff --git a/dom0/sd-vm-updates.top b/dom0/sd-vm-updates.top deleted file mode 100644 index f3b391d5..00000000 --- a/dom0/sd-vm-updates.top +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -# "Placeholder" config to trigger TemplateVM boots, -# so upgrades can be applied automatically via cron. -base: - qubes:type:template: - - match: pillar - - topd diff --git a/dom0/sd-whonix-hidserv-key.top b/dom0/sd-whonix-hidserv-key.top deleted file mode 100644 index 5bc0fd52..00000000 --- a/dom0/sd-whonix-hidserv-key.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - sd-whonix: - - sd-whonix-hidserv-key diff --git a/dom0/sd-whonix.top b/dom0/sd-whonix.top deleted file mode 100644 index c334ecb5..00000000 --- a/dom0/sd-whonix.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-whonix diff --git a/dom0/sd-workstation-template-files.top b/dom0/sd-workstation-template-files.top deleted file mode 100644 index 5e177838..00000000 --- a/dom0/sd-workstation-template-files.top +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - securedrop-workstation: - - fpf-apt-test-repo - - sd-workstation-template-files diff --git a/dom0/sd-workstation-template.top b/dom0/sd-workstation-template.top deleted file mode 100644 index 084f97fe..00000000 --- a/dom0/sd-workstation-template.top +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -base: - dom0: - - sd-workstation-template diff --git a/dom0/sd-workstation.top b/dom0/sd-workstation.top new file mode 100644 index 00000000..eb8811e1 --- /dev/null +++ b/dom0/sd-workstation.top @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# vim: set syntax=yaml ts=2 sw=2 sts=2 et : + +base: + dom0: + - sd-sys-vms + - sd-dom0-files + - sd-workstation-template + - sd-dom0-qvm-rpc + - sd-export + - sd-gpg + - sd-proxy + - sd-svs-disp + - sd-svs + - sd-whonix + sd-export-template: + - sd-export-files + sd-gpg: + - sd-gpg-files + sd-proxy-template: + - sd-proxy-template-files + sd-svs: + - sd-svs-config + sd-svs-disp-template: + - sd-svs-disp-files + sd-svs-template: + - sd-svs-files + sys-firewall: + - sd-sys-firewall-files + sd-whonix: + - sd-whonix-hidserv-key + securedrop-workstation: + - sd-workstation-template-files + + # "Placeholder" config to trigger TemplateVM boots, + # so upgrades can be applied automatically via cron. + qubes:type:template: + - match: pillar + - topd From e540ea6c10ee9471692b0a0de4b8141c43d2a945 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Thu, 21 Nov 2019 17:03:06 -0800 Subject: [PATCH 3/6] More robust clean action Ensures that the various system files configured on dom0 and sys-firewall as part of installation are removed as part of the "make clean" action. The file cleanups are implemented in Salt. The VM destruction has been ported from Bash to Python (via the Qubes Admin API). The removal logic is smarter: it only removes VMs that have the tag "sd-workstation", and removes TemplateVMs last, to avoid ordering problems in the execution of `qvm-remove`. --- Makefile | 9 +-- dom0/sd-clean-all.sls | 45 +++++++++++++ dom0/sd-svs-config.sls | 2 + dom0/sd-sys-firewall-files.sls | 3 + scripts/destroy-vm | 119 +++++++++++++++++++++++---------- 5 files changed, 137 insertions(+), 41 deletions(-) create mode 100644 dom0/sd-clean-all.sls diff --git a/Makefile b/Makefile index 8d92c6aa..5c71114a 100644 --- a/Makefile +++ b/Makefile @@ -75,10 +75,10 @@ remove-sd-export: assert-dom0 ## Destroys SD EXPORT VMs @./scripts/destroy-vm sd-export-usb @./scripts/destroy-vm sd-export-usb-dvm -clean: assert-dom0 destroy-all clean-salt ## Destroys all SD VMs +clean: assert-dom0 prep-salt destroy-all ## Destroys all SD VMs + sudo qubesctl --show-output state.sls sd-clean-all sudo dnf -y -q remove securedrop-workstation-dom0-config 2>/dev/null || true - sudo rm -f /usr/bin/securedrop-update \ - /etc/cron.daily/securedrop-update-cron + $(MAKE) clean-salt test: assert-dom0 ## Runs all application tests (no integration tests yet) python3 -m unittest discover -v tests @@ -126,7 +126,8 @@ list-vms: ## Prints all Qubes VMs managed by Workstation salt config @./scripts/list-vms destroy-all: ## Destroys all VMs managed by Workstation salt config - @./scripts/list-vms | xargs ./scripts/destroy-vm + qubes-prefs default_dispvm fedora-30-dvm + ./scripts/destroy-vm --all .PHONY: update-pip-requirements update-pip-requirements: ## Updates all Python requirements files via pip-compile. diff --git a/dom0/sd-clean-all.sls b/dom0/sd-clean-all.sls new file mode 100644 index 00000000..e4e94749 --- /dev/null +++ b/dom0/sd-clean-all.sls @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# vim: set syntax=yaml ts=2 sw=2 sts=2 et : + +set-fedora-as-default-dispvm: + cmd.run: + - name: qubes-prefs default_dispvm fedora-30-dvm + +remove-dom0-sdw-config-files: + file.absent: + - names: + - /opt/securedrop + - /etc/yum.repos.d/securedrop-workstation-dom0.repo + - /usr/bin/securedrop-update + - /etc/pki/rpm-gpg/RPM-GPG-KEY-securedrop-workstation-test + - /etc/cron.daily/securedrop-update-cron + - /usr/share/securedrop/icons + +sd-cleanup-sys-firewall: + cmd.run: + - names: + - qvm-run sys-firewall 'sudo rm -f /rw/config/RPM-GPG-KEY-securedrop-workstation-test' + - qvm-run sys-firewall 'sudo rm -f /rw/config/sd-copy-rpm-repo-pubkey.sh' + - qvm-run sys-firewall 'sudo rm -f /etc/pki/rpm-gpg/RPM-GPG-KEY-securedrop-workstation-test' + - qvm-run sys-firewall 'sudo perl -pi -E "s#^/rw/config/sd-copy-rpm-repo-pubkey.sh##" /rw/config/rc.local' + +sd-cleanup-rpc-mgmt-policy: + file.replace: + - names: + - /etc/qubes-rpc/policy/qubes.VMShell + - /etc/qubes-rpc/policy/qubes.VMRootShell + - repl: '' + - pattern: '^disp-mgmt-sd-\w+\s+sd-\w+\s+allow,user=root' + +{% set sdw_customized_rpc_files = salt['cmd.shell']('grep -rl "BEGIN securedrop-workstation" /etc/qubes-rpc/ | cat').splitlines() %} +{% if sdw_customized_rpc_files|length > 0 %} +sd-cleanup-rpc-policy-grants: + file.replace: + - names: {{ sdw_customized_rpc_files }} + - pattern: '### BEGIN securedrop-workstation ###.*### END securedrop-workstation ###' + - flags: + - MULTILINE + - DOTALL + - repl: '' + - backup: no +{% endif %} diff --git a/dom0/sd-svs-config.sls b/dom0/sd-svs-config.sls index dd72aa1d..2c5689ea 100644 --- a/dom0/sd-svs-config.sls +++ b/dom0/sd-svs-config.sls @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: set syntax=yaml ts=2 sw=2 sts=2 et : ## # sd-svs-config # ======== diff --git a/dom0/sd-sys-firewall-files.sls b/dom0/sd-sys-firewall-files.sls index e75d3083..0b1468c0 100644 --- a/dom0/sd-sys-firewall-files.sls +++ b/dom0/sd-sys-firewall-files.sls @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +# vim: set syntax=yaml ts=2 sw=2 sts=2 et : +# sys-firewall-rpm-test-key: file.managed: - name: /rw/config/RPM-GPG-KEY-securedrop-workstation-test diff --git a/scripts/destroy-vm b/scripts/destroy-vm index d7036883..099ad859 100755 --- a/scripts/destroy-vm +++ b/scripts/destroy-vm @@ -1,37 +1,82 @@ -#!/bin/bash -# Utility to quickly destroy a Qubes VM managed by the Workstation -# salt config, for use in repeated builds during development. -# -# There are a few "sleep"s sprinkled around because the qubes tooling -# returns very quickly, often before the underlying operation has -# completed. -set -e -set -u - - -if [[ $# -lt 1 ]]; then - printf "Usage: %s [vm_name ...]\n" "$0" - exit 1 -fi - -function destroy-qubes-vm() { - local vm_name="$1" - printf "Destroying VM '%s'... " "$vm_name" - if qvm-check "$vm_name" > /dev/null 2>&1 ; then - # Wait a sec, otherwise qubesd may return empty - sleep 1 - if qvm-check --running "$vm_name" > /dev/null 2>&1 ; then - sleep 1 - qvm-kill "$vm_name" > /dev/null 2>&1 - sleep 1 - fi - qvm-remove -f "$vm_name" > /dev/null 2>&1 - printf "OK\n" - else - printf "(absent)\n" - fi -} - -for vm_name in $@; do - destroy-qubes-vm "$vm_name" -done +#!/usr/bin/env python3 +""" +Utility to quickly destroy a Qubes VM managed by the Workstation +salt config, for use in repeated builds during development. +""" +import argparse +import subprocess +import sys + +import qubesadmin + + +SDW_DEFAULT_TAG = "sd-workstation" + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--all", + default=False, + required=False, + action="store_true", + help="Destroys all SDW VMs", + ) + parser.add_argument( + "targets", + help="List of SDW VMs to destroy", + nargs=argparse.REMAINDER, + ) + args = parser.parse_args() + if not args.all and len(args.targets) < 1: + parser.print_help(sys.stderr) + sys.exit(1) + return args + + +def destroy_vm(vm): + """ + Destroys a single VM instance. Requires arg to be + QubesVM object. + """ + assert SDW_DEFAULT_TAG in vm.tags + if vm.is_running(): + vm.kill() + print("Destroying VM '{}'... ".format(vm.name), end="") + subprocess.check_call("qvm-remove -f {}".format(vm.name).split()) + print("OK") + + +def destroy_all(): + """ + Destroys all VMs marked with the 'sd-workstation' tag, in the following order: + DispVMs, AppVMs, then TemplateVMs. Excludes VMs for which + installed_by_rpm=true. + """ + # Remove DispVMs first, then AppVMs, then TemplateVMs last. + sdw_vms = [vm for vm in q.domains if SDW_DEFAULT_TAG in vm.tags] + sdw_template_vms = [vm for vm in sdw_vms + if vm.klass == "TemplateVM" and not + vm.installed_by_rpm] + sdw_disp_vms = [vm for vm in sdw_vms if vm.klass == "DispVM"] + sdw_app_vms = [vm for vm in sdw_vms if vm.klass == "AppVM"] + + for vm in sdw_disp_vms: + destroy_vm(vm) + + for vm in sdw_app_vms: + destroy_vm(vm) + + for vm in sdw_template_vms: + destroy_vm(vm) + + +if __name__ == "__main__": + args = parse_args() + q = qubesadmin.Qubes() + if args.all: + destroy_all() + else: + for t in args.targets: + vm = q.domains[t] + destroy_vm(vm) From 63edaede3247b26e408d23ca176253b7a4b1808d Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Thu, 21 Nov 2019 17:03:52 -0800 Subject: [PATCH 4/6] Includes upstream update.qubes-vm formula Rather than duplicating logic to mitigate apt CVEs, let's use the Qubes-maintained Salt calls to manage VM upgrades. The reuse will be particularly helpful as we revisit the unattended-vm-upgrades-via-cron logic. --- dom0/fpf-apt-test-repo.sls | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/dom0/fpf-apt-test-repo.sls b/dom0/fpf-apt-test-repo.sls index 776d59e0..bbea3185 100644 --- a/dom0/fpf-apt-test-repo.sls +++ b/dom0/fpf-apt-test-repo.sls @@ -1,19 +1,8 @@ # -*- coding: utf-8 -*- # vim: set syntax=yaml ts=2 sw=2 sts=2 et : - -# Handle misconfigured jessie-backports repo in default debian-9 TemplateVM. -# The Jessie repos aren't maintained anymore, and their inclusion causes -# even apt update to fail. -remove-jessie-backports-repo: - file.line: - - name: /etc/apt/sources.list - # Unclear why "Delete" *must* be capitalized, but that's the case! - - mode: delete - - match: jessie-backports - # quiet param seems to be ignored, so using "onlyif" to test existence - - quiet: True - - onlyif: - - test -f /etc/apt/sources.list +# +include: + - update.qubes-vm # That's right, we need to install a package in order to # configure a repo to install another package @@ -22,9 +11,10 @@ install-python-apt-for-repo-config: - pkgs: - python-apt - require: - - file: remove-jessie-backports-repo + # Require that the Qubes update state has run first + - sls: update.qubes-vm -configure apt-test apt repo: +configure-apt-test-apt-repo: pkgrepo.managed: - name: "deb [arch=amd64] https://apt-test-qubes.freedom.press stretch main" - file: /etc/apt/sources.list.d/securedrop_workstation.list @@ -32,12 +22,3 @@ configure apt-test apt repo: - clean_file: True # squash file to ensure there are no duplicates - require: - pkg: install-python-apt-for-repo-config - -# Ensure all apt updates are applied, since the VMs -# will be cloned, duplicating package version drift. -update-all-apt-packages: - pkg.uptodate: - - cache_valid_time: "3600" - - dist_upgrade: True - - require: - - pkg: install-python-apt-for-repo-config From 1fede0efe84ded8a7bdff0cfc3f78272f334d8f2 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Sat, 23 Nov 2019 21:04:31 -0800 Subject: [PATCH 5/6] Leaner provision-all Removes hardcoding of VM names to leverage bits of the Admin API (via `qvm-ls --tags`) where appropriate. Updated comments throughout. Trying to lean on Salt's parallel execution as much as possible, to keep runtimes low. The step-by-step execution of Salt (sys-firewall, then dom0, then VMs) could be handled by Salt orchestrators, but not implementing such a substantial refactor right now. [0] https://docs.saltstack.com/en/latest/topics/orchestrate/orchestrate_runner.html --- Makefile | 3 --- scripts/list-vms | 28 ---------------------------- scripts/provision-all | 27 ++++++++++++--------------- 3 files changed, 12 insertions(+), 46 deletions(-) delete mode 100755 scripts/list-vms diff --git a/Makefile b/Makefile index 5c71114a..850106cc 100644 --- a/Makefile +++ b/Makefile @@ -122,9 +122,6 @@ prep-dom0: prep-salt # Copies dom0 config files for VM updates sudo qubesctl top.enable sd-dom0-files sudo qubesctl --show-output --targets dom0 state.highstate -list-vms: ## Prints all Qubes VMs managed by Workstation salt config - @./scripts/list-vms - destroy-all: ## Destroys all VMs managed by Workstation salt config qubes-prefs default_dispvm fedora-30-dvm ./scripts/destroy-vm --all diff --git a/scripts/list-vms b/scripts/list-vms deleted file mode 100755 index 2407f822..00000000 --- a/scripts/list-vms +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Prints names of all Salt-managed Workstation VMs. -# Intended for use with other actions, e.g. destroy-all. -set -e -set -u -set -o pipefail - - -# When adding new VMs, ensure the template is listed *after* the AppVMs that -# use it. -declare -a sd_workstation_vm_names=( - sd-gpg - sd-proxy - sd-proxy-template - sd-svs - sd-svs-template - securedrop-workstation - sd-whonix - sd-svs-disp - sd-svs-disp-template - sd-export-usb - sd-export-usb-dvm - sd-export-template -) - -for vm in "${sd_workstation_vm_names[@]}" ; do - echo "$vm" -done diff --git a/scripts/provision-all b/scripts/provision-all index 46d77c04..96f8830f 100755 --- a/scripts/provision-all +++ b/scripts/provision-all @@ -5,25 +5,22 @@ set -u set -o pipefail -# Format list of all VMs comma-separated, for use as qubesctl target -all_sdw_vms_target="$(./scripts/list-vms | perl -npE 's/\n/,/g' | perl -npE 's/,$//' )" - +# The max concurrency reduction (4->2) was required to avoid "did not return clean data" +# errors from qubesctl. It may be possible to raise this again. +max_concurrency="2" echo "Configure Fedora-based system VMs" -sudo qubesctl --show-output --targets dom0 state.sls sd-sys-vms +sudo qubesctl --show-output state.sls sd-sys-vms sudo qubesctl --show-output --skip-dom0 --targets sys-firewall state.sls sd-sys-firewall-files -echo "Create base Template to be used by others" -sudo qubesctl --show-output --targets dom0 state.sls sd-workstation-template +# Running only against dom0, to ensure the VMs are created (but not yet configured) +echo "Set up dom0 config files, including RPC policies, and create VMs" +sudo qubesctl --show-output state.highstate -echo "Configure packages inside base Template" -sudo qubesctl --show-output --skip-dom0 --targets securedrop-workstation state.sls sd-workstation-template-files +# Format list of all VMs comma-separated, for use as qubesctl target +# We run this after dom0's highstate, so that the VMs are available for listing by tag. +all_sdw_vms_target="$(qvm-ls --tags sd-workstation --raw-list | perl -npE 's/\n/,/g' | perl -npE 's/,$//' )" -echo "Set up dom0 config files, including RPC policies, and create VMs" -# The dom0 config runs implicitly via qubesctl (unless `--skip-dom0` is passed), so the VM -# creation logic will be run before the states adding files inside the VMs. -#sudo qubesctl --show-output --targets dom0 state.highstate +# We skip dom0 in the task below, since dom0 highstate was enforced in the previous command. echo "Provision all SecureDrop Workstation VMs with service-specific configs" -# The max concurrency reduction (4->2) was required to avoid "did not return clean data" -# errors from qubesctl. It may be possible to raise this again. -sudo qubesctl --show-output --max-concurrency 2 --targets "$all_sdw_vms_target" state.highstate +sudo qubesctl --show-output --max-concurrency "$max_concurrency" --skip-dom0 --targets "$all_sdw_vms_target" state.highstate From 1b7a901c945b2044afebe584109ed7817f15d283 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Mon, 2 Dec 2019 09:11:52 -0800 Subject: [PATCH 6/6] Address review comments Hat tip to @emkll. Specifically: * Don't process binary files with grep in dom0 * Makes destroy-vm subprocess cmd more readable * Adds comments to use of "update.qubes-vm" SLS * Removes extraneous newlines from RPC policy cleanup --- dom0/fpf-apt-test-repo.sls | 9 ++++++++- dom0/sd-clean-all.sls | 4 ++-- scripts/destroy-vm | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/dom0/fpf-apt-test-repo.sls b/dom0/fpf-apt-test-repo.sls index bbea3185..6163a546 100644 --- a/dom0/fpf-apt-test-repo.sls +++ b/dom0/fpf-apt-test-repo.sls @@ -1,6 +1,12 @@ # -*- coding: utf-8 -*- # vim: set syntax=yaml ts=2 sw=2 sts=2 et : # + +# Import the Qubes-maintained Salt logic for upgrading VM packages. +# Intelligently handles both Debian & Fedora VMs. For reference, see: +# +# dom0:/srv/formulas/base/update-formula/update/qubes-vm.sls +# include: - update.qubes-vm @@ -11,7 +17,8 @@ install-python-apt-for-repo-config: - pkgs: - python-apt - require: - # Require that the Qubes update state has run first + # Require that the Qubes update state has run first. Doing so + # will ensure that apt is sufficiently patched prior to installing. - sls: update.qubes-vm configure-apt-test-apt-repo: diff --git a/dom0/sd-clean-all.sls b/dom0/sd-clean-all.sls index e4e94749..167370a5 100644 --- a/dom0/sd-clean-all.sls +++ b/dom0/sd-clean-all.sls @@ -31,12 +31,12 @@ sd-cleanup-rpc-mgmt-policy: - repl: '' - pattern: '^disp-mgmt-sd-\w+\s+sd-\w+\s+allow,user=root' -{% set sdw_customized_rpc_files = salt['cmd.shell']('grep -rl "BEGIN securedrop-workstation" /etc/qubes-rpc/ | cat').splitlines() %} +{% set sdw_customized_rpc_files = salt['cmd.shell']('grep -rIl "BEGIN securedrop-workstation" /etc/qubes-rpc/ | cat').splitlines() %} {% if sdw_customized_rpc_files|length > 0 %} sd-cleanup-rpc-policy-grants: file.replace: - names: {{ sdw_customized_rpc_files }} - - pattern: '### BEGIN securedrop-workstation ###.*### END securedrop-workstation ###' + - pattern: '### BEGIN securedrop-workstation ###.*### END securedrop-workstation ###\s*' - flags: - MULTILINE - DOTALL diff --git a/scripts/destroy-vm b/scripts/destroy-vm index 099ad859..a0d65a81 100755 --- a/scripts/destroy-vm +++ b/scripts/destroy-vm @@ -43,7 +43,7 @@ def destroy_vm(vm): if vm.is_running(): vm.kill() print("Destroying VM '{}'... ".format(vm.name), end="") - subprocess.check_call("qvm-remove -f {}".format(vm.name).split()) + subprocess.check_call(["qvm-remove", "-f", vm.name]) print("OK")