Merge pull request #5199 from freedomofpress/update-ansible

Update ansible to 2.9.7

admin/requirements-ansible.in

@@ -1,3 +1,3 @@
-ansible>2.7<2.8
+ansible>2.9.7<2.10
 cryptography>=2.7
 netaddr

admin/requirements.txt

@@ -4,43 +4,12 @@
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements-ansible.in requirements.in
 #
-ansible==2.7.13 \
-    --hash=sha256:339c87a1bf9e8555ce1e1c1a9452d8ed1df240944ec1a3fc2e813e6c7d70aeae
+ansible==2.9.7 \
+    --hash=sha256:7222ce925536a25b2912364e13b03a3e21dbf2f96799ebff304f48509324de7b
 asn1crypto==0.24.0 \
     --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \
     --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 \
     # via cryptography
-bcrypt==3.1.4 \
-    --hash=sha256:01477981abf74e306e8ee31629a940a5e9138de000c6b0898f7f850461c4a0a5 \
-    --hash=sha256:054d6e0acaea429e6da3613fcd12d05ee29a531794d96f6ab959f29a39f33391 \
-    --hash=sha256:0872eeecdf9a429c1420158500eedb323a132bc5bf3339475151c52414729e70 \
-    --hash=sha256:09a3b8c258b815eadb611bad04ca15ec77d86aa9ce56070e1af0d5932f17642a \
-    --hash=sha256:0f317e4ffbdd15c3c0f8ab5fbd86aa9aabc7bea18b5cc5951b456fe39e9f738c \
-    --hash=sha256:2788c32673a2ad0062bea850ab73cffc0dba874db10d7a3682b6f2f280553f20 \
-    --hash=sha256:321d4d48be25b8d77594d8324c0585c80ae91ac214f62db9098734e5e7fb280f \
-    --hash=sha256:346d6f84ff0b493dbc90c6b77136df83e81f903f0b95525ee80e5e6d5e4eef84 \
-    --hash=sha256:34dd60b90b0f6de94a89e71fcd19913a30e83091c8468d0923a93a0cccbfbbff \
-    --hash=sha256:3b4c23300c4eded8895442c003ae9b14328ae69309ac5867e7530de8bdd7875d \
-    --hash=sha256:43d1960e7db14042319c46925892d5fa99b08ff21d57482e6f5328a1aca03588 \
-    --hash=sha256:49e96267cd9be55a349fd74f9852eb9ae2c427cd7f6455d0f1765d7332292832 \
-    --hash=sha256:67ed1a374c9155ec0840214ce804616de49c3df9c5bc66740687c1c9b1cd9e8d \
-    --hash=sha256:6efd9ca20aefbaf2e7e6817a2c6ed4a50ff6900fafdea1bcb1d0e9471743b144 \
-    --hash=sha256:8569844a5d8e1fdde4d7712a05ab2e6061343ac34af6e7e3d7935b2bd1907bfd \
-    --hash=sha256:8629ea6a8a59f865add1d6a87464c3c676e60101b8d16ef404d0a031424a8491 \
-    --hash=sha256:988cac675e25133d01a78f2286189c1f01974470817a33eaf4cfee573cfb72a5 \
-    --hash=sha256:9a6fedda73aba1568962f7543a1f586051c54febbc74e87769bad6a4b8587c39 \
-    --hash=sha256:9eced8962ce3b7124fe20fd358cf8c7470706437fa064b9874f849ad4c5866fc \
-    --hash=sha256:a005ed6163490988711ff732386b08effcbf8df62ae93dd1e5bda0714fad8afb \
-    --hash=sha256:ae35dbcb6b011af6c840893b32399252d81ff57d52c13e12422e16b5fea1d0fb \
-    --hash=sha256:b1e8491c6740f21b37cca77bc64677696a3fb9f32360794d57fa8477b7329eda \
-    --hash=sha256:c906bdb482162e9ef48eea9f8c0d967acceb5c84f2d25574c7d2a58d04861df1 \
-    --hash=sha256:cb18ffdc861dbb244f14be32c47ab69604d0aca415bee53485fcea4f8e93d5ef \
-    --hash=sha256:d86da365dda59010ba0d1ac45aa78390f56bf7f992e65f70b3b081d5e5257b09 \
-    --hash=sha256:e22f0997622e1ceec834fd25947dc2ee2962c2133ea693d61805bc867abaf7ea \
-    --hash=sha256:f2fe545d27a619a552396533cddf70d83cecd880a611cdfdbb87ca6aec52f66b \
-    --hash=sha256:f7fd3ed3745fe6e81e28dc3b3d76cce31525a91f32a387e1febd6b982caf8cdb \
-    --hash=sha256:f9210820ee4818d84658ed7df16a7f30c9fba7d8b139959950acef91745cc0f7 \
-    # via paramiko
 cffi==1.11.4 \
     --hash=sha256:0640f12f04f257c4467075a804a4920a5d07ef91e11c525fc65d715c08231c81 \
     --hash=sha256:0fe3b3d571543a4065059d1d3d6d39f4ca6da0f2207ad13547094522e32ead46 \
@@ -69,7 +38,7 @@ cffi==1.11.4 \
     --hash=sha256:ec208ca16e57904dd7f4c7568665f80b1f7eb7e3214be014560c28def219060d \
     --hash=sha256:f4719d0bafc5f0a67b2ec432086d40f653840698d41fa6e9afa679403dea9d78 \
     --hash=sha256:f4992cd7b4c867f453d44c213ee29e8fd484cf81cfece4b6e836d0982b6fa1cf \
-    # via bcrypt, cryptography, pynacl
+    # via cryptography
 cryptography==2.7 \
     --hash=sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c \
     --hash=sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643 \
@@ -128,46 +97,13 @@ markupsafe==1.1.1 \
 netaddr==0.7.19 \
     --hash=sha256:38aeec7cdd035081d3a4c306394b19d677623bf76fa0913f6695127c7753aefd \
     --hash=sha256:56b3558bd71f3f6999e4c52e349f38660e54a7a8a9943335f73dfc96883e08ca
-paramiko==2.4.2 \
-    --hash=sha256:3c16b2bfb4c0d810b24c40155dbfd113c0521e7e6ee593d704e84b4c658a1f3b \
-    --hash=sha256:a8975a7df3560c9f1e2b43dc54ebd40fd00a7017392ca5445ce7df409f900fcb \
-    # via ansible
 prompt_toolkit==2.0.9 \
     --hash=sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780 \
     --hash=sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1 \
     --hash=sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55
-pyasn1==0.4.2 \
-    --hash=sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15 \
-    --hash=sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1 \
-    # via paramiko
 pycparser==2.18 \
     --hash=sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226 \
     # via cffi
-pynacl==1.2.1 \
-    --hash=sha256:04e30e5bdeeb2d5b34107f28cd2f5bbfdc6c616f3be88fc6f53582ff1669eeca \
-    --hash=sha256:0bfa0d94d2be6874e40f896e0a67e290749151e7de767c5aefbad1121cad7512 \
-    --hash=sha256:11aa4e141b2456ce5cecc19c130e970793fa3a2c2e6fbb8ad65b28f35aa9e6b6 \
-    --hash=sha256:13bdc1fe084ff9ac7653ae5a924cae03bf4bb07c6667c9eb5b6eb3c570220776 \
-    --hash=sha256:14339dc233e7a9dda80a3800e64e7ff89d0878ba23360eea24f1af1b13772cac \
-    --hash=sha256:1d33e775fab3f383167afb20b9927aaf4961b953d76eeb271a5703a6d756b65b \
-    --hash=sha256:2a42b2399d0428619e58dac7734838102d35f6dcdee149e0088823629bf99fbb \
-    --hash=sha256:2dce05ac8b3c37b9e2f65eab56c544885607394753e9613fd159d5e2045c2d98 \
-    --hash=sha256:6453b0dae593163ffc6db6f9c9c1597d35c650598e2c39c0590d1757207a1ac2 \
-    --hash=sha256:73a5a96fb5fbf2215beee2353a128d382dbca83f5341f0d3c750877a236569ef \
-    --hash=sha256:8abb4ef79161a5f58848b30ab6fb98d8c466da21fdd65558ce1d7afc02c70b5f \
-    --hash=sha256:8ac1167195b32a8755de06efd5b2d2fe76fc864517dab66aaf65662cc59e1988 \
-    --hash=sha256:8f505f42f659012794414fa57c498404e64db78f1d98dfd40e318c569f3c783b \
-    --hash=sha256:be71cd5fce04061e1f3d39597f93619c80cdd3558a6c9ba99a546f144a8d8101 \
-    --hash=sha256:cf6877124ae6a0698404e169b3ba534542cfbc43f939d46b927d956daf0a373a \
-    --hash=sha256:d0eb5b2795b7ee2cbcfcadacbe95a13afbda048a262bd369da9904fecb568975 \
-    --hash=sha256:d795f506bcc9463efb5ebb0f65ed77921dcc9e0a50499dedd89f208445de9ecb \
-    --hash=sha256:d8aaf7e5d6b0e0ef7d6dbf7abeb75085713d0100b4eb1a4e4e857de76d77ac45 \
-    --hash=sha256:e0d38fa0a75f65f556fb912f2c6790d1fa29b7dd27a1d9cc5591b281321eaaa9 \
-    --hash=sha256:eb2acabbd487a46b38540a819ef67e477a674481f84a82a7ba2234b9ba46f752 \
-    --hash=sha256:eeee629828d0eb4f6d98ac41e9a3a6461d114d1d0aa111a8931c049359298da0 \
-    --hash=sha256:f5ce9e26d25eb0b2d96f3ef0ad70e1d3ae89b5d60255c462252a3e456a48c053 \
-    --hash=sha256:fabf73d5d0286f9e078774f3435601d2735c94ce9e514ac4fb945701edead7e4 \
-    # via paramiko
 pyyaml==5.3.1 \
     --hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
     --hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
@@ -183,7 +119,7 @@ pyyaml==5.3.1 \
 six==1.11.0 \
     --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
     --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb \
-    # via bcrypt, cryptography, prompt-toolkit, pynacl
+    # via cryptography, prompt-toolkit
 wcwidth==0.1.7 \
     --hash=sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e \
     --hash=sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c \

ansible.cfg

@@ -1,4 +1,5 @@
 [defaults]
+interpreter_python=auto
 #ask_pass=False
 #ask_sudo_pass=False
 host_key_checking=False

install_files/ansible-base/ansible.cfg

@@ -1,4 +1,5 @@
 [defaults]
+interpreter_python=auto
 #ask_pass=False
 #ask_sudo_pass=False
 display_skipped_hosts=False

install_files/ansible-base/callback_plugins/ansible_version_check.py

@@ -19,13 +19,18 @@ def print_red_bold(text):
 
 class CallbackModule(CallbackBase):
     def __init__(self):
-        # Can't use `on_X` because this isn't forwards compatible
-        # with Ansible 2.0+
-        required_version = '2.7.13'  # Keep synchronized with requirements files
-        if not ansible.__version__.startswith(required_version):
+        # The acceptable version range needs to be synchronized with
+        # requirements files.
+        viable_start = [2, 9, 7]
+        viable_end = [2, 10, 0]
+        ansible_version = [int(v) for v in ansible.__version__.split('.')]
+        if not (viable_start <= ansible_version < viable_end):
             print_red_bold(
-                "SecureDrop restriction: only Ansible {version}.*"
-                "is supported."
-                .format(version=required_version)
+                "SecureDrop restriction: Ansible version must be at least {viable_start} "
+                "and less than {viable_end}."
+                .format(
+                    viable_start='.'.join(str(v) for v in viable_start),
+                    viable_end='.'.join(str(v) for v in viable_end),
+                )
             )
             sys.exit(1)

install_files/ansible-base/group_vars/all/securedrop

@@ -13,7 +13,7 @@ install_local_packages: false
 ci_builder: false
 
 # Ansible v1 default reference to remote host
-remote_host_ref: "{{ ansible_ssh_host|default(inventory_hostname) }}"
+remote_host_ref: "{{ ansible_host|default(inventory_hostname) }}"
 
 # Packages required for working on SecureDrop within development VM. Same list
 # is used on the build VM to pull in required packages.

install_files/ansible-base/inventory

@@ -6,6 +6,5 @@
 # 1. Set up HidServAuth in your torrc with the values from app-ssh-aths and mon-ssh-aths
 # 2. Replace the IP addresses here with the corresponding .onion addresses
 #
-app ansible_ssh_host=10.20.1.2 ansible_ssh_port=22
-mon ansible_ssh_host=10.20.2.2 ansible_ssh_port=22
-
+app ansible_host=10.20.1.2 ansible_port=22
+mon ansible_host=10.20.2.2 ansible_port=22

install_files/ansible-base/inventory-dynamic

@@ -182,8 +182,8 @@ def build_inventory():
         "_meta": {
             "hostvars": {
                 h: {
-                    "ansible_ssh_host": lookup_ssh_address(h),
-                    "ansible_ssh_user": lookup_admin_username(),
+                    "ansible_host": lookup_ssh_address(h),
+                    "ansible_user": lookup_admin_username(),
                 } for h in SECUREDROP_SUPPORTED_HOSTNAMES
             },
         },

install_files/ansible-base/roles/build-ossec-deb-pkg/tasks/main.yml

@@ -12,7 +12,7 @@
 
 - name: Install python-requests for URL fetching
   apt:
-    name: python-requests
+    name: python3-requests
     state: present
     update_cache: yes
     cache_valid_time: 3600

install_files/ansible-base/roles/common/tasks/apt_upgrade.yml

@@ -29,7 +29,7 @@
     # We only want to prevent upgrades if we're connected over Tor; first-time
     # installs will happen over direct access SSH (to local IPv4 address).
     # If we're connecting to an Onion URL, then it's over Tor.
-    - (ansible_host|default(ansible_ssh_host)).endswith('.onion')
+    - (ansible_host|default(ansible_host)).endswith('.onion')
 
 - name: Perform safe upgrade to ensure all the packages are updated.
   apt:

install_files/ansible-base/roles/tor-hidden-services/handlers/main.yml

@@ -1,3 +1,3 @@
 ---
 - name: restart tor
-  include: restart-tor-carefully.yml
+  include_tasks: "{{ role_path }}/handlers/restart-tor-carefully.yml"

install_files/ansible-base/roles/tor-hidden-services/handlers/restart-tor-carefully.yml

@@ -1,29 +1,10 @@
 ---
-# Meta handler to bounce tor service sanely. If the SSH connection
-# for the remote host is over Tor (i.e. host ends in `.onion`), then
-# bounce the tor service via fire-and-forget, wait, then reestablish
-# the connection after polling for the service to come back up.
 
-# Registering a concise variable for use in conditionals, essentially
-# deciding "Are we connected via SSH over Tor or not?"
-- name: Register host name to wait for.
-  set_fact:
-    _hostname_to_wait_for: "{{ remote_host_ref|default(ansible_host) }}"
-
-# If we're not connected over Tor, bounce the service as usual.
-- name: restart tor (simple)
-  service:
-    name: tor
-    state: restarted
-  when: not _hostname_to_wait_for.endswith('.onion')
-
-# As of Ansible v2.2, the `service` module is not compatible with the `async`
-# parameter. This was changed in 2.3.
-- name: restart tor (async)
+- name: restart tor service (async)
   shell: sleep 5 && service tor restart
   async: 3000
   poll: 0
-  when: _hostname_to_wait_for.endswith('.onion')
+
 
 # It's critical that we eliminate existing SSH connections, otherwise Ansible
 # will try to reuse the stale ControlPersist files, which no longer have an
@@ -37,13 +18,12 @@
       state: absent
   run_once: yes
   become: no
-  when: _hostname_to_wait_for.endswith('.onion')
 
 - name: Waiting for SSH connection (slow)...
   local_action: wait_for
   args:
-    host: "{{ _hostname_to_wait_for }}"
-    port: "{{ ansible_ssh_port|default(ansible_port|default(22)) }}"
+    host: "{{ remote_host_ref|default(ansible_host) }}"
+    port: "{{ ansible_port|default(ansible_port|default(22)) }}"
     delay: 30
     search_regex: OpenSSH
     state: started

install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml

@@ -36,4 +36,4 @@
     - tor_instances_v3|length > 0
     # Only run if we're connected over Tor (i.e. enabling v3 after v2).
     # If we're not connected over Tor, this is a first-run.
-    - (ansible_host|default(ansible_ssh_host)).endswith('.onion')
+    - (ansible_host|default(ansible_host)).endswith('.onion')

install_files/ansible-base/tasks/transistion_ssh_local.yml

@@ -24,12 +24,12 @@
   shell: sleep 2 && shutdown -r now
   async: 1
   poll: 0
-  when: aths_deletion_results|changed
+  when: aths_deletion_results.changed
 
 - name: Provide helpful user message and end early
   fail:
     msg: |
       Due to the transition from ssh-over-tor to ssh-over-localnet
       please run `./securedrop-admin tailsconfig` and then re-run
       `./securedrop-admin install` again to continue re-configuration.
-  when: aths_deletion_results|changed
+  when: aths_deletion_results.changed

molecule/ansible-config/molecule.yml

@@ -20,6 +20,7 @@ provisioner:
   config_options:
     defaults:
       callback_whitelist: "profile_tasks, timer"
+      interpreter_python: auto
   inventory:
     links:
       group_vars: ../../install_files/ansible-base/group_vars

molecule/builder-xenial/molecule.yml

@@ -39,6 +39,9 @@ provisioner:
   inventory:
     links:
       group_vars: ../../install_files/ansible-base/group_vars
+  config_options:
+    defaults:
+      interpreter_python: auto
   options:
     e: "@ansible-override-vars.yml"
   env:

molecule/fetch-tor-packages/molecule.yml

@@ -12,6 +12,9 @@ provisioner:
   name: ansible
   lint:
     name: ansible-lint
+  config_options:
+    defaults:
+      interpreter_python: auto
 scenario:
   name: fetch-tor-packages
   test_sequence:

molecule/libvirt-staging-xenial/molecule.yml

@@ -43,6 +43,9 @@ provisioner:
   name: ansible
   lint:
     name: ansible-lint
+  config_options:
+    defaults:
+      interpreter_python: auto
   options:
     e: "@ansible-override-vars.yml"
   playbooks:

molecule/qubes-staging/create.yml

@@ -37,7 +37,7 @@
     - name: Wait for VMs to boot
       pause:
         seconds: 15
-      when: start_result|changed
+      when: start_result.changed
 
     - name: Get IP address for instances
       command: qvm-ls --raw-data --field ip {{ item.vm_name }}

molecule/qubes-staging/molecule.yml

@@ -31,6 +31,7 @@ provisioner:
   config_options:
     defaults:
       callback_whitelist: "profile_tasks, timer"
+      interpreter_python: auto
   options:
     e: "@qubes-vars.yml"
   playbooks:

molecule/shared/sd_clone.yml

@@ -25,6 +25,18 @@
         src: "{{ molecule_ephemeral_directory }}/{{ sd_clone_dir }}/{{ sd_ansible_roles }}"
         state: link
         force: yes
+
+    # Required for mutual compatibility during Ansible 2.7.x -> 2.9.x transition
+    # We moved to 2.9.x in v1.3.0, so don't run if historical version is >= to 1.3.0.
+    - name: Patch tor handler
+      lineinfile:
+        dest: "{{ molecule_ephemeral_directory }}/{{ sd_clone_dir }}/{{ sd_ansible_roles }}/tor-hidden-services/handlers/main.yml"
+        regexp: '^  include:'
+        line: '  include_tasks: "{% raw %}{{ role_path }}{% endraw %}/handlers/restart-tor-carefully.yml"'
+      when: >
+        ORIG_SECUREDROP_VER.startswith("1.2") or
+        ORIG_SECUREDROP_VER.startswith("1.1") or
+        ORIG_SECUREDROP_VER.startswith("1.0")
   vars:
     sd_clone_dir: "sd-orig"
     sd_ansible_roles: "install_files/ansible-base/roles"

molecule/upgrade/molecule.yml

@@ -58,6 +58,8 @@ provisioner:
   options:
     e: "@ansible-override-vars.yml"
   config_options:
+    defaults:
+      interpreter_python: auto
     ssh_connection:
       pipelining: True
   playbooks:

molecule/vagrant-packager/molecule.yml

@@ -48,6 +48,8 @@ provisioner:
     skip-tags: "local_build"
     e: "@ansible-override-vars.yml"
   config_options:
+    defaults:
+      interpreter_python: auto
     ssh_connection:
       pipelining: True
   playbooks:

molecule/virtualbox-staging-xenial/molecule.yml

@@ -37,6 +37,9 @@ provisioner:
   name: ansible
   lint:
     name: ansible-lint
+  config_options:
+    defaults:
+      interpreter_python: auto
   options:
     e: "@ansible-override-vars.yml"
   playbooks:

securedrop/requirements/python3/develop-requirements.in

@@ -1,3 +1,4 @@
+ansible>=2.9.7,<2.10.0
 bandit
 # yes, we need both boto and boto3
 boto