From 5a24f645625f10865ebce6b246049cc2768fd15f Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Wed, 26 Aug 2020 16:40:58 -0700 Subject: [PATCH 1/5] Begins libvirt-staging-focal scenario Starting with a naive copy/paste from the libvirt-staging-xenial scenario. Will tweak settings as required to get it to run. --- .../ansible-override-vars.yml | 7 ++ molecule/libvirt-staging-focal/create.yml | 56 ++++++++++++++ molecule/libvirt-staging-focal/destroy.yml | 35 +++++++++ molecule/libvirt-staging-focal/molecule.yml | 74 +++++++++++++++++++ molecule/libvirt-staging-focal/prepare.yml | 9 +++ 5 files changed, 181 insertions(+) create mode 100644 molecule/libvirt-staging-focal/ansible-override-vars.yml create mode 100644 molecule/libvirt-staging-focal/create.yml create mode 100644 molecule/libvirt-staging-focal/destroy.yml create mode 100644 molecule/libvirt-staging-focal/molecule.yml create mode 100644 molecule/libvirt-staging-focal/prepare.yml diff --git a/molecule/libvirt-staging-focal/ansible-override-vars.yml b/molecule/libvirt-staging-focal/ansible-override-vars.yml new file mode 100644 index 0000000000..05c029a0ca --- /dev/null +++ b/molecule/libvirt-staging-focal/ansible-override-vars.yml @@ -0,0 +1,7 @@ +--- +# Permit direct access via SSH +ssh_net_in_override: 0.0.0.0/0 + +# In libvirt, we want to connect over eth0, not eth1 which is used for +# inter-VM communication for OSSEC. +ssh_ip: "{{ ansible_default_ipv4.address }}" diff --git a/molecule/libvirt-staging-focal/create.yml b/molecule/libvirt-staging-focal/create.yml new file mode 100644 index 0000000000..ffe945c5c2 --- /dev/null +++ b/molecule/libvirt-staging-focal/create.yml @@ -0,0 +1,56 @@ +--- +- name: Create + hosts: localhost + connection: local + vars: + molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" + molecule_instance_config: "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}" + molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}" + tasks: + + - name: Create molecule instance(s) + molecule_vagrant: + instance_name: "{{ item.name }}" + instance_interfaces: "{{ item.interfaces | default(omit) }}" + instance_raw_config_args: "{{ item.instance_raw_config_args | default(omit) }}" + + platform_box: "{{ item.box }}" + platform_box_version: "{{ item.box_version | default(omit) }}" + platform_box_url: "{{ item.box_url | default(omit) }}" + + provider_name: "{{ molecule_yml.driver.provider.name }}" + provider_memory: "{{ item.memory | default(omit) }}" + provider_cpus: "{{ item.cpus | default(omit) }}" + provider_raw_config_args: "{{ item.raw_config_args | default(omit) }}" + force_stop: yes + + state: up + register: server + with_items: "{{ molecule_yml.platforms }}" + + # Mandatory configuration for Molecule to function. + + - name: Populate instance config dict + set_fact: + instance_conf_dict: { + 'instance': "{{ item.Host }}", + 'address': "{{ item.HostName }}", + 'user': "{{ item.User }}", + 'port': "{{ item.Port }}", + 'identity_file': "{{ item.IdentityFile }}", } + with_items: "{{ server.results }}" + register: instance_config_dict + when: server.changed | bool + + - name: Convert instance config dict to a list + set_fact: + instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}" + when: server.changed | bool + + - name: Dump instance config + copy: + # NOTE(retr0h): Workaround for Ansible 2.2. + # https://github.com/ansible/ansible/issues/20885 + content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" + dest: "{{ molecule_instance_config }}" + when: server.changed | bool diff --git a/molecule/libvirt-staging-focal/destroy.yml b/molecule/libvirt-staging-focal/destroy.yml new file mode 100644 index 0000000000..c287e47d88 --- /dev/null +++ b/molecule/libvirt-staging-focal/destroy.yml @@ -0,0 +1,35 @@ +--- + +- name: Destroy + hosts: localhost + connection: local + vars: + molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" + molecule_instance_config: "{{ lookup('env',' MOLECULE_INSTANCE_CONFIG') }}" + molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}" + molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + tasks: + - name: Destroy molecule instance(s) + molecule_vagrant: + instance_name: "{{ item.name }}" + platform_box: "{{ item.box }}" + provider_name: "{{ molecule_yml.driver.provider.name }}" + force_stop: "{{ item.force_stop | default(True) }}" + + state: destroy + register: server + with_items: "{{ molecule_yml.platforms }}" + + # Mandatory configuration for Molecule to function. + + - name: Populate instance config + set_fact: + instance_conf: {} + + - name: Dump instance config + copy: + # NOTE(retr0h): Workaround for Ansible 2.2. + # https://github.com/ansible/ansible/issues/20885 + content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" + dest: "{{ molecule_instance_config }}" + when: server.changed | bool diff --git a/molecule/libvirt-staging-focal/molecule.yml b/molecule/libvirt-staging-focal/molecule.yml new file mode 100644 index 0000000000..d02a7e386c --- /dev/null +++ b/molecule/libvirt-staging-focal/molecule.yml @@ -0,0 +1,74 @@ +--- +driver: + name: vagrant + provider: + name: libvirt +lint: + name: yamllint + +platforms: + - name: app-staging + box: bento/ubuntu-16.04 + raw_config_args: + - "cpu_mode = 'host-passthrough'" + - "video_type = 'virtio'" + instance_raw_config_args: + - "vm.synced_folder './', '/vagrant', disabled: true" + - "vm.network 'private_network', ip: '10.0.1.2'" + - "ssh.insert_key = false" + memory: 1024 + private_ip: 10.0.1.2 + groups: + - securedrop_application_server + - securedrop + - staging + + - name: mon-staging + box: bento/ubuntu-16.04 + raw_config_args: + - "cpu_mode = 'host-passthrough'" + - "video_type = 'virtio'" + instance_raw_config_args: + - "vm.synced_folder './', '/vagrant', disabled: true" + - "vm.network 'private_network', ip: '10.0.1.3'" + - "ssh.insert_key = false" + memory: 1024 + private_ip: 10.0.1.3 + groups: + - securedrop_monitor_server + - securedrop + - staging + +provisioner: + name: ansible + lint: + name: ansible-lint + config_options: + defaults: + interpreter_python: auto + options: + e: "@ansible-override-vars.yml" + playbooks: + converge: ../../install_files/ansible-base/securedrop-staging.yml + create: create.yml + destroy: destroy.yml + prepare: prepare.yml + env: + ANSIBLE_CONFIG: ../../install_files/ansible-base/ansible.cfg + +scenario: + name: libvirt-staging-xenial + test_sequence: + - destroy + - create + - converge + - verify +verifier: + name: testinfra + lint: + name: flake8 + directory: ../testinfra/staging/ + options: + n: auto + v: 2 + junit-xml: ../../junit/testinfra-results.xml diff --git a/molecule/libvirt-staging-focal/prepare.yml b/molecule/libvirt-staging-focal/prepare.yml new file mode 100644 index 0000000000..112da19982 --- /dev/null +++ b/molecule/libvirt-staging-focal/prepare.yml @@ -0,0 +1,9 @@ +--- +- name: Prepare + hosts: all + gather_facts: False + tasks: + - name: Install python for Ansible + raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) + become: True + changed_when: False From ae95a93e2bd9d62f97ae7d9014e7ac3bd5b81fa3 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Wed, 26 Aug 2020 16:41:37 -0700 Subject: [PATCH 2/5] Focal scenario: updates references xenial -> focal The bare minimum required to get the copy/pasted Xenial libvirt scenario working under Focal. Commiting separately to make the changes a bit easier to inspect. --- molecule/libvirt-staging-focal/ansible-override-vars.yml | 3 +++ molecule/libvirt-staging-focal/molecule.yml | 6 +++--- molecule/libvirt-staging-focal/prepare.yml | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/molecule/libvirt-staging-focal/ansible-override-vars.yml b/molecule/libvirt-staging-focal/ansible-override-vars.yml index 05c029a0ca..75ed126093 100644 --- a/molecule/libvirt-staging-focal/ansible-override-vars.yml +++ b/molecule/libvirt-staging-focal/ansible-override-vars.yml @@ -5,3 +5,6 @@ ssh_net_in_override: 0.0.0.0/0 # In libvirt, we want to connect over eth0, not eth1 which is used for # inter-VM communication for OSSEC. ssh_ip: "{{ ansible_default_ipv4.address }}" + +# Make sure Focal packages are used during installation +securedrop_staging_install_target_distro: "focal" diff --git a/molecule/libvirt-staging-focal/molecule.yml b/molecule/libvirt-staging-focal/molecule.yml index d02a7e386c..e7f0844461 100644 --- a/molecule/libvirt-staging-focal/molecule.yml +++ b/molecule/libvirt-staging-focal/molecule.yml @@ -8,7 +8,7 @@ lint: platforms: - name: app-staging - box: bento/ubuntu-16.04 + box: bento/ubuntu-20.04 raw_config_args: - "cpu_mode = 'host-passthrough'" - "video_type = 'virtio'" @@ -24,7 +24,7 @@ platforms: - staging - name: mon-staging - box: bento/ubuntu-16.04 + box: bento/ubuntu-20.04 raw_config_args: - "cpu_mode = 'host-passthrough'" - "video_type = 'virtio'" @@ -57,7 +57,7 @@ provisioner: ANSIBLE_CONFIG: ../../install_files/ansible-base/ansible.cfg scenario: - name: libvirt-staging-xenial + name: libvirt-staging-focal test_sequence: - destroy - create diff --git a/molecule/libvirt-staging-focal/prepare.yml b/molecule/libvirt-staging-focal/prepare.yml index 112da19982..64571aaebd 100644 --- a/molecule/libvirt-staging-focal/prepare.yml +++ b/molecule/libvirt-staging-focal/prepare.yml @@ -4,6 +4,6 @@ gather_facts: False tasks: - name: Install python for Ansible - raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) + raw: test -e /usr/bin/python3 || (apt -y update && apt install -y python3-minimal) become: True changed_when: False From 773012d4dfde281b0a40758960c29a94c5a08055 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Fri, 4 Sep 2020 09:23:43 -0700 Subject: [PATCH 3/5] Uses platform-specific vars in common role In order to ease the transition to Focal, let's use different vars for packages depending on platform. When running the Xenial logic against Focal, the kernel removal tasks failed since several of the package names were not found. Also adding a "grsecurity" tag since the grsec story for Focal still needs work, so it's easy to skip now. --- .../ansible-base/roles/common/defaults/main.yml | 9 +++++++++ .../ansible-base/roles/common/tasks/install_ntp.yml | 8 -------- .../roles/common/tasks/install_packages.yml | 7 +++++++ .../ansible-base/roles/common/tasks/install_tmux.yml | 9 --------- install_files/ansible-base/roles/common/tasks/main.yml | 6 +++--- .../roles/common/tasks/remove_unused_packages.yml | 10 ++-------- .../ansible-base/roles/common/vars/Ubuntu_focal.yml | 4 ++++ .../ansible-base/roles/common/vars/Ubuntu_xenial.yml | 9 +++++++++ 8 files changed, 34 insertions(+), 28 deletions(-) delete mode 100644 install_files/ansible-base/roles/common/tasks/install_ntp.yml create mode 100644 install_files/ansible-base/roles/common/tasks/install_packages.yml delete mode 100644 install_files/ansible-base/roles/common/tasks/install_tmux.yml create mode 100644 install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml create mode 100644 install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml diff --git a/install_files/ansible-base/roles/common/defaults/main.yml b/install_files/ansible-base/roles/common/defaults/main.yml index a9de7da68a..1c2dda22b8 100644 --- a/install_files/ansible-base/roles/common/defaults/main.yml +++ b/install_files/ansible-base/roles/common/defaults/main.yml @@ -3,6 +3,15 @@ # and aid in clearing memory. Only the hour is configurable. daily_reboot_time: 4 # An integer between 0 and 23 +securedrop_common_packages: + - apt-transport-https + - aptitude + - cron-apt + - ntp + - ntpdate + - resolvconf + - tmux + disabled_kernel_modules: - btusb - bluetooth diff --git a/install_files/ansible-base/roles/common/tasks/install_ntp.yml b/install_files/ansible-base/roles/common/tasks/install_ntp.yml deleted file mode 100644 index 3e8b390e8d..0000000000 --- a/install_files/ansible-base/roles/common/tasks/install_ntp.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: Install ntp for ntpd. - apt: - pkg: ['ntp', 'ntpdate'] - state: present - tags: - - apt - - ntp diff --git a/install_files/ansible-base/roles/common/tasks/install_packages.yml b/install_files/ansible-base/roles/common/tasks/install_packages.yml new file mode 100644 index 0000000000..5d48223a82 --- /dev/null +++ b/install_files/ansible-base/roles/common/tasks/install_packages.yml @@ -0,0 +1,7 @@ +--- +- name: Install base apt depedencies + apt: + name: "{{ securedrop_common_packages }}" + state: present + update_cache: yes + cache_valid_time: 3600 diff --git a/install_files/ansible-base/roles/common/tasks/install_tmux.yml b/install_files/ansible-base/roles/common/tasks/install_tmux.yml deleted file mode 100644 index 09638b53ea..0000000000 --- a/install_files/ansible-base/roles/common/tasks/install_tmux.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Install tmux. - apt: - pkg: tmux - state: present - update_cache: yes - cache_valid_time: 3600 - tags: - - apt diff --git a/install_files/ansible-base/roles/common/tasks/main.yml b/install_files/ansible-base/roles/common/tasks/main.yml index 04d7cd881e..a2904ba03e 100644 --- a/install_files/ansible-base/roles/common/tasks/main.yml +++ b/install_files/ansible-base/roles/common/tasks/main.yml @@ -1,5 +1,7 @@ --- -- include: install_ntp.yml +- include_vars: "{{ ansible_distribution }}_{{ ansible_distribution_release }}.yml" + +- include: install_packages.yml - include: post_ubuntu_install_checks.yml @@ -9,8 +11,6 @@ - include: harden_dns.yml -- include: install_tmux.yml - - include: cron_apt.yml tags: - reboot diff --git a/install_files/ansible-base/roles/common/tasks/remove_unused_packages.yml b/install_files/ansible-base/roles/common/tasks/remove_unused_packages.yml index 57b52a5913..d0a91c8bd1 100644 --- a/install_files/ansible-base/roles/common/tasks/remove_unused_packages.yml +++ b/install_files/ansible-base/roles/common/tasks/remove_unused_packages.yml @@ -13,14 +13,7 @@ # We must used command due to the use of wildcards - name: Remove generic kernel packages. command: apt-get remove -y {{ item }} - with_items: - - linux-signed-generic - - linux-signed-generic-lts-utopic - - linux-signed-image-generic - - linux-signed-image-generic-lts-utopic - - linux-image-generic-lts-xenial - - 'linux-image-.*generic' - - 'linux-headers-.*' + with_items: "{{ securedrop_kernel_packages_to_remove }}" register: apt_removed_kernels changed_when: "'The following packages will be REMOVED' in apt_removed_kernels.stdout" tags: @@ -41,6 +34,7 @@ with_items: "{{ apt_installed_kernels.stdout_lines }}" tags: - apt + - grsecurity - name: Remove dependencies that are no longer required apt: diff --git a/install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml b/install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml new file mode 100644 index 0000000000..8c7c48b509 --- /dev/null +++ b/install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml @@ -0,0 +1,4 @@ +--- +securedrop_kernel_packages_to_remove: + - linux-virtual + - linux-generic diff --git a/install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml b/install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml new file mode 100644 index 0000000000..ccc4279155 --- /dev/null +++ b/install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml @@ -0,0 +1,9 @@ +--- +securedrop_kernel_packages_to_remove: + - linux-signed-generic + - linux-signed-generic-lts-utopic + - linux-signed-image-generic + - linux-signed-image-generic-lts-utopic + - linux-image-generic-lts-xenial + - 'linux-image-.*generic' + - 'linux-headers-.*' From d0ddfc5af44206086efc66f202f9b7ccd1d51723 Mon Sep 17 00:00:00 2001 From: John Hensley Date: Mon, 14 Sep 2020 17:06:05 -0400 Subject: [PATCH 4/5] Update tor upgrade check for newer aptitude Between the version of aptitude on Xenial and that on Focal, empty searches started returning 1, causing `Check whether tor will be upgraded` to fail. Let's ignore the exit code, since what we care about is whether "tor" is in the stdout. We could use `apt list --upgradable`, but that command warns about lack of a stable CLI, and additionally we'd have to munge the output further to inspect it. --- install_files/ansible-base/roles/common/tasks/apt_upgrade.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install_files/ansible-base/roles/common/tasks/apt_upgrade.yml b/install_files/ansible-base/roles/common/tasks/apt_upgrade.yml index 0c7a5c1a70..06d5788619 100644 --- a/install_files/ansible-base/roles/common/tasks/apt_upgrade.yml +++ b/install_files/ansible-base/roles/common/tasks/apt_upgrade.yml @@ -20,6 +20,8 @@ command: aptitude search '~U' --display-format '%p' --disable-columns register: tor_upgradable_result changed_when: false + # aptitude >= 0.7.6 will exit non-zero if no hits + failed_when: false - name: Hold tor package to prevent upgrade breaking SSH connection. command: aptitude hold tor From 58e3be21ac8ce9028f73646d1439d5e7344b4303 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Mon, 14 Sep 2020 16:51:24 -0700 Subject: [PATCH 5/5] Adds "staging-focal" target Configures a libvirt VM staging environment based on Focal. Far from complete, but it's a start. --- Makefile | 9 +++++++-- devops/scripts/select-staging-env | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c6a524b4c4..0d00414cb5 100644 --- a/Makefile +++ b/Makefile @@ -195,11 +195,16 @@ docs: ## Build project documentation with live reload for editing. @echo .PHONY: staging -staging: ## Create a local staging environment in virtual machines. - @echo "███ Creating staging environment..." +staging: ## Create a local staging environment in virtual machines (Xenial) + @echo "███ Creating staging environment on Ubuntu Xenial..." @$(SDROOT)/devops/scripts/create-staging-env @echo +.PHONY: staging-focal +staging-focal: ## Create a local staging environment in virtual machines (Focal) + @echo "███ Creating staging environment on Ubuntu Focal..." + @$(SDROOT)/devops/scripts/create-staging-env focal + @echo .PHONY: testinfra testinfra: ## Run infra tests against a local staging environment. diff --git a/devops/scripts/select-staging-env b/devops/scripts/select-staging-env index 75138ad04e..98483bb6cf 100755 --- a/devops/scripts/select-staging-env +++ b/devops/scripts/select-staging-env @@ -11,7 +11,9 @@ set -e set -o pipefail -securedrop_platform_suffix="-xenial" + +# Support overrides for LTS version +securedrop_platform_suffix="-${1:-xenial}" # Respect explicit choice of Vagrant provider if set. if [[ -n "${VAGRANT_DEFAULT_PROVIDER:-}" ]] ; then