diff --git a/.circleci/config.yml b/.circleci/config.yml index 90cafc9aa2..bf8c00b2aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -328,7 +328,7 @@ jobs: BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs)") if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi make ci-go - no_output_timeout: 20m + no_output_timeout: 25m - run: name: Ensure environment torn down diff --git a/.gitignore b/.gitignore index 66107224f8..1daefbeab8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,12 +15,21 @@ wheelhouse # ignore the instance information JSON file to prevent commit of private info securedrop/tests/functional/instance_information.json +# ignore v3 onion JSON file +install_files/ansible-base/tor_v3_keys.json + # ignore the ATHS/THS hostname file ansible places +# Tor v2 app-ssh-aths app-document-aths # leave this here for historic reasons app-journalist-aths app-source-ths mon-ssh-aths +# Tor v3 +app-journalist.auth_private +app-sourcev3-ths +app-ssh.auth_private +mon-ssh.auth_private *.key *.csr *.pem diff --git a/Makefile b/Makefile index fcc3b2cecd..cad651e579 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ update-python3-requirements: ## Update Python 3 requirements with pip-compile. @$(DEVSHELL) pip-compile \ --output-file requirements/python3/develop-requirements.txt \ ../admin/requirements-ansible.in \ + ../admin/requirements.in \ requirements/python3/develop-requirements.in @$(DEVSHELL) pip-compile \ --output-file requirements/python3/test-requirements.txt \ @@ -51,6 +52,7 @@ update-python2-requirements: ## Update Python 2 requirements with pip-compile. @PYTHON_VERSION=2 $(DEVSHELL) pip-compile \ --output-file requirements/python2/develop-requirements.txt \ ../admin/requirements-ansible.in \ + ../admin/requirements.in \ requirements/python2/develop-requirements.in @PYTHON_VERSION=2 $(DEVSHELL) pip-compile \ --output-file requirements/python2/test-requirements.txt \ @@ -60,7 +62,7 @@ update-python2-requirements: ## Update Python 2 requirements with pip-compile. requirements/python2/securedrop-app-code-requirements.in .PHONY: update-pip-requirements -update-pip-requirements: update-admin-pip-requirements update-python3-requirements ## Update all requirements with pip-compile. +update-pip-requirements: update-admin-pip-requirements update-python2-requirements update-python3-requirements ## Update all requirements with pip-compile. ################# diff --git a/admin/Dockerfile b/admin/Dockerfile index 934a77cf1f..58c3213cc1 100644 --- a/admin/Dockerfile +++ b/admin/Dockerfile @@ -17,4 +17,7 @@ ENV VIRTUAL_ENV /opt/.venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY requirements-dev.txt . RUN pip install --require-hashes -r requirements-dev.txt +# Now also pin pip due to https://github.com/jazzband/pip-tools/issues/853 +RUN pip install pip==19.1 + RUN chown -R $USER_NAME /opt diff --git a/admin/requirements-ansible.in b/admin/requirements-ansible.in index f7a963fe08..09201692c6 100644 --- a/admin/requirements-ansible.in +++ b/admin/requirements-ansible.in @@ -1,3 +1,3 @@ ansible>2.6<2.7 -cryptography>=2.3 +cryptography>=2.7 netaddr diff --git a/admin/requirements.txt b/admin/requirements.txt index c6f4324bfc..36d933bcd0 100644 --- a/admin/requirements.txt +++ b/admin/requirements.txt @@ -70,36 +70,29 @@ cffi==1.11.4 \ --hash=sha256:f4719d0bafc5f0a67b2ec432086d40f653840698d41fa6e9afa679403dea9d78 \ --hash=sha256:f4992cd7b4c867f453d44c213ee29e8fd484cf81cfece4b6e836d0982b6fa1cf \ # via bcrypt, cryptography, pynacl -cryptography==2.3 \ - --hash=sha256:21af753934f2f6d1a10fe8f4c0a64315af209ef6adeaee63ca349797d747d687 \ - --hash=sha256:27bb401a20a838d6d0ea380f08c6ead3ccd8c9d8a0232dc9adcc0e4994576a66 \ - --hash=sha256:29720c4253263cff9aea64585adbbe85013ba647f6e98367efff9db2d7193ded \ - --hash=sha256:2a35b7570d8f247889784010aac8b384fd2e4a47b33e15c4a60b45a7c1944120 \ - --hash=sha256:42c531a6a354407f42ee07fda5c2c0dc822cf6d52744949c182f2b295fbd4183 \ - --hash=sha256:5eb86f03f9c4f0ac2336ac5431271072ddf7ecc76b338e26366732cfac58aa19 \ - --hash=sha256:67f7f57eae8dede577f3f7775957f5bec93edd6bdb6ce597bb5b28e1bdf3d4fb \ - --hash=sha256:6ec84edcbc966ae460560a51a90046503ff0b5b66157a9efc61515c68059f6c8 \ - --hash=sha256:7ba834564daef87557e7fcd35c3c3183a4147b0b3a57314e53317360b9b201b3 \ - --hash=sha256:7d7f084cbe1fdb82be5a0545062b59b1ad3637bc5a48612ac2eb428ff31b31ea \ - --hash=sha256:82409f5150e529d699e5c33fa8fd85e965104db03bc564f5f4b6a9199e591f7c \ - --hash=sha256:87d092a7c2a44e5f7414ab02fb4145723ebba411425e1a99773531dd4c0e9b8d \ - --hash=sha256:8c56ef989342e42b9fcaba7c74b446f0cc9bed546dd00034fa7ad66fc00307ef \ - --hash=sha256:9449f5d4d7c516a6118fa9210c4a00f34384cb1d2028672100ee0c6cce49d7f6 \ - --hash=sha256:bc2301170986ad82d9349a91eb8884e0e191209c45f5541b16aa7c0cfb135978 \ - --hash=sha256:c132bab45d4bd0fff1d3fe294d92b0a6eb8404e93337b3127bdec9f21de117e6 \ - --hash=sha256:c3d945b7b577f07a477700f618f46cbc287af3a9222cd73035c6ef527ef2c363 \ - --hash=sha256:cee18beb4c807b5c0b178f4fa2fae03cef9d51821a358c6890f8b23465b7e5d2 \ - --hash=sha256:d01dfc5c2b3495184f683574e03c70022674ca9a7be88589c5aba130d835ea90 +cryptography==2.7 \ + --hash=sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c \ + --hash=sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643 \ + --hash=sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216 \ + --hash=sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799 \ + --hash=sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a \ + --hash=sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9 \ + --hash=sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc \ + --hash=sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8 \ + --hash=sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53 \ + --hash=sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1 \ + --hash=sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609 \ + --hash=sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292 \ + --hash=sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e \ + --hash=sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6 \ + --hash=sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed \ + --hash=sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d enum34==1.1.6 \ --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \ --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \ --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \ --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \ # via cryptography -idna==2.6 \ - --hash=sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f \ - --hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 \ - # via cryptography ipaddress==1.0.19 \ --hash=sha256:200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81 \ # via cryptography diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py index 720b9b28bd..7f068707f9 100755 --- a/admin/securedrop_admin/__init__.py +++ b/admin/securedrop_admin/__init__.py @@ -33,10 +33,14 @@ import subprocess import sys import types +import json +import base64 import prompt_toolkit from prompt_toolkit.validation import Validator, ValidationError import yaml from pkg_resources import parse_version +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import x25519 sdlog = logging.getLogger(__name__) RELEASE_KEY = '22245C81E3BAEB4138B36061310F561200F4AD77' @@ -566,6 +570,73 @@ def sdconfig(args): return 0 +def generate_new_v3_keys(): + """This function generate new keys for Tor v3 onion + services and returns them as as tuple. + + :returns: Tuple(public_key, private_key) + """ + + private_key = x25519.X25519PrivateKey.generate() + private_bytes = private_key.private_bytes( + encoding=serialization.Encoding.Raw , + format=serialization.PrivateFormat.Raw, + encryption_algorithm=serialization.NoEncryption()) + public_key = private_key.public_key() + public_bytes = public_key.public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw) + + # Base32 encode and remove base32 padding characters (`=`) + # Using try/except blocks for Python 2/3 support. + try: + public = base64.b32encode(public_bytes).replace('=', '') \ + .decode("utf-8") + except TypeError: + public = base64.b32encode(public_bytes).replace(b'=', b'') \ + .decode("utf-8") + try: + private = base64.b32encode(private_bytes).replace('=', '') \ + .decode("utf-8") + except TypeError: + private = base64.b32encode(private_bytes).replace(b'=', b'') \ + .decode("utf-8") + return public, private + + +def find_or_generate_new_torv3_keys(args): + """ + This method will either read v3 Tor onion service keys if found or generate + a new public/private keypair. + """ + secret_key_path = os.path.join(args.ansible_path, + "tor_v3_keys.json") + if os.path.exists(secret_key_path): + print('Tor v3 onion service keys already exist in: {}'.format( + secret_key_path)) + return 0 + # No old keys, generate and store them first + app_journalist_public_key, \ + app_journalist_private_key = generate_new_v3_keys() + # For app ssh service + app_ssh_public_key, app_ssh_private_key = generate_new_v3_keys() + # For mon ssh service + mon_ssh_public_key, mon_ssh_private_key = generate_new_v3_keys() + tor_v3_service_info = { + "app_journalist_public_key": app_journalist_public_key, + "app_journalist_private_key": app_journalist_private_key, + "app_ssh_public_key": app_ssh_public_key, + "app_ssh_private_key": app_ssh_private_key, + "mon_ssh_public_key": mon_ssh_public_key, + "mon_ssh_private_key": mon_ssh_private_key, + } + with open(secret_key_path, 'w') as fobj: + json.dump(tor_v3_service_info, fobj, indent=4) + print('Tor v3 onion service keys generated and stored in: {}'.format( + secret_key_path)) + return 0 + + def install_securedrop(args): """Install/Update SecureDrop""" SiteConfig(args).load() @@ -827,6 +898,11 @@ class ArgParseFormatterCombo(argparse.ArgumentDefaultsHelpFormatter, help=run_tails_config.__doc__) parse_tailsconfig.set_defaults(func=run_tails_config) + parse_generate_tor_keys = subparsers.add_parser( + 'generate_v3_keys', + help=find_or_generate_new_torv3_keys.__doc__) + parse_generate_tor_keys.set_defaults(func=find_or_generate_new_torv3_keys) + parse_backup = subparsers.add_parser('backup', help=backup_securedrop.__doc__) parse_backup.set_defaults(func=backup_securedrop) diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py index 31b34c1d6a..2300701222 100644 --- a/admin/tests/test_securedrop-admin.py +++ b/admin/tests/test_securedrop-admin.py @@ -22,6 +22,7 @@ import argparse from flaky import flaky from os.path import dirname, join, basename, exists +import json import mock from prompt_toolkit.validation import ValidationError import pytest @@ -1008,3 +1009,58 @@ def test_load(self, caplog): with pytest.raises(yaml.YAMLError) as e: site_config.load() assert 'issue processing' in caplog.text + + +def test_generate_new_v3_keys(): + public, private = securedrop_admin.generate_new_v3_keys() + + for key in [public, private]: + # base32 padding characters should be removed + assert '=' not in key + assert len(key) == 52 + + +def test_find_or_generate_new_torv3_keys_first_run(tmpdir, capsys): + args = argparse.Namespace(ansible_path=str(tmpdir)) + + return_code = securedrop_admin.find_or_generate_new_torv3_keys(args) + + captured = capsys.readouterr() + assert 'Tor v3 onion service keys generated' in captured.out + assert return_code == 0 + + secret_key_path = os.path.join(args.ansible_path, + "tor_v3_keys.json") + + with open(secret_key_path) as f: + v3_onion_service_keys = json.load(f) + + expected_keys = ['app_journalist_public_key', + 'app_journalist_private_key', + 'app_ssh_public_key', + 'app_ssh_private_key', + 'mon_ssh_public_key', + 'mon_ssh_private_key'] + for key in expected_keys: + assert key in v3_onion_service_keys.keys() + + +def test_find_or_generate_new_torv3_keys_subsequent_run(tmpdir, capsys): + args = argparse.Namespace(ansible_path=str(tmpdir)) + + secret_key_path = os.path.join(args.ansible_path, + "tor_v3_keys.json") + old_keys = {'foo': 'bar'} + with open(secret_key_path, 'w') as f: + json.dump(old_keys, f) + + return_code = securedrop_admin.find_or_generate_new_torv3_keys(args) + + captured = capsys.readouterr() + assert 'Tor v3 onion service keys already exist' in captured.out + assert return_code == 0 + + with open(secret_key_path) as f: + v3_onion_service_keys = json.load(f) + + assert v3_onion_service_keys == old_keys diff --git a/install_files/ansible-base/group_vars/all/securedrop b/install_files/ansible-base/group_vars/all/securedrop index b797cbab9b..0fbe7663a6 100644 --- a/install_files/ansible-base/group_vars/all/securedrop +++ b/install_files/ansible-base/group_vars/all/securedrop @@ -46,7 +46,6 @@ appserver_dependencies: # Enable Tor over SSH by default enable_ssh_over_tor: true - # If file is present on system at the end of ansible run # force a reboot. Needed because of the de-coupled nature of # the many roles of the current prod playbook diff --git a/install_files/ansible-base/group_vars/securedrop_application_server.yml b/install_files/ansible-base/group_vars/securedrop_application_server.yml index 762efd6f87..3874a95c1a 100644 --- a/install_files/ansible-base/group_vars/securedrop_application_server.yml +++ b/install_files/ansible-base/group_vars/securedrop_application_server.yml @@ -20,6 +20,17 @@ tor_instances: - service: journalist filename: app-journalist-aths +tor_instances_v3: + - "{{ {'service': 'sshv3', 'filename': 'app-ssh.auth_private'} if enable_ssh_over_tor else [] }}" + - service: sourcev3 + filename: app-sourcev3-ths + - service: journalistv3 + filename: app-journalist.auth_private + +tor_auth_instances_v3: + - "{{ 'sshv3' if enable_ssh_over_tor else [] }}" + - "journalistv3" + authd_iprules: - chain: OUTPUT dest: "{{ monitor_ip }}" diff --git a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml index dca829e96a..b646d02af2 100644 --- a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml +++ b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml @@ -14,6 +14,10 @@ local_deb_packages: # Configure the tor Onion Services. The Monitor server has only one, # for SSH, since no web interfaces. tor_instances: "{{ [{ 'service': 'ssh', 'filename': 'mon-ssh-aths'}] if enable_ssh_over_tor else [] }}" +tor_instances_v3: "{{ [{ 'service': 'sshv3', 'filename': 'mon-ssh.auth_private'}] if enable_ssh_over_tor else [] }}" + +tor_auth_instances_v3: + - "{{ 'sshv3' if enable_ssh_over_tor else [] }}" authd_iprules: - chain: INPUT diff --git a/install_files/ansible-base/group_vars/staging.yml b/install_files/ansible-base/group_vars/staging.yml index ea8d764d3c..d13ac7d012 100644 --- a/install_files/ansible-base/group_vars/staging.yml +++ b/install_files/ansible-base/group_vars/staging.yml @@ -50,6 +50,9 @@ postfix_enable_service: no # Otherwise, all SSH connections would be forced over Tor. enable_ssh_over_tor: false +# v3 onion services should be available in staging for testing. +v3_onion_services: true + ### Use for backup restores ### # If the `backup_zip` variable is defined ansible will copy the defined file to # the app server and run the 0.3_collect.py script to unzip and restore those diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.tor b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.tor index 5a2c373f8a..50a75f686a 100644 --- a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.tor +++ b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.tor @@ -51,6 +51,22 @@ /var/lib/tor/services/hostname.tmp rw, /var/lib/tor/sevices/private_key rw, /var/lib/tor/services/private_key.tmp rw, + /var/lib/tor/services/sourcev3/ w, + /var/lib/tor/services/sourcev3/hostname rw, + /var/lib/tor/services/sourcev3/hs_ed25519_public_key rw, + /var/lib/tor/services/sourcev3/hs_ed25519_secret_key rw, + /var/lib/tor/services/journalistv3/ w, + /var/lib/tor/services/journalistv3/hostname rw, + /var/lib/tor/services/journalistv3/hs_ed25519_public_key rw, + /var/lib/tor/services/journalistv3/hs_ed25519_secret_key rw, + /var/lib/tor/services/journalistv3/authorized_keys/ w, + /var/lib/tor/services/journalistv3/authorized_keys/client.auth r, + /var/lib/tor/services/sshv3/ w, + /var/lib/tor/services/sshv3/hostname rw, + /var/lib/tor/services/sshv3/hs_ed25519_public_key rw, + /var/lib/tor/services/sshv3/hs_ed25519_secret_key rw, + /var/lib/tor/services/sshv3/authorized_keys/ w, + /var/lib/tor/services/sshv3/authorized_keys/client.auth r, /var/lib/tor/lock rwk, /var/lib/tor/state rw, /var/lib/tor/state.tmp rw, diff --git a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml index 8028884f60..73d3c8917b 100644 --- a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml +++ b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml @@ -11,7 +11,6 @@ ssh_listening_address: "{{ '127.0.0.1' if enable_ssh_over_tor else '0.0.0.0' }}" # the hostname files can be fetched back to the Admin Workstation. tor_hidden_services_parent_dir: /var/lib/tor/services - # Platform specific command for inferring the local interface utilized # to route to a specific IP host admin_net_int: @@ -21,3 +20,23 @@ admin_net_int: Darwin: cmd: "/sbin/route -n get " rgx: "(?<=interface: )\\w+" + +# Whether to fetch back client-auth settings from the remote hosts. +# We make this conditional to support disabling during dynamic role includes, +# required for the ssh-over-lan strategy. +fetch_tor_client_auth_configs: true + +# v2 Tor onion services are on / v3 Tor onion services are off by default for backwards +# compatibility. Note that new install after 1.0 will have v3 enabled by sdconfig which +# will override these variables. +v2_onion_services: true +v3_onion_services: false + +# Lookup table for querying keypair info from the local JSON +# file on the Admin Workstation, required for configuring client +# auth on Tor v3 Onion URLs. See the tor_v3_keys.json file for +# reference on structure. +tor_v3_service_map: + app-journalist.auth_private: "app_journalist_private_key" + app-ssh.auth_private: "app_ssh_private_key" + mon-ssh.auth_private: "mon_ssh_private_key" diff --git a/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml b/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml index 76b278370d..499de3650a 100644 --- a/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml +++ b/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml @@ -5,6 +5,7 @@ path: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}/hostname" delay: 5 with_items: "{{ tor_instances }}" + when: "v2_onion_services" tags: - tor @@ -14,6 +15,7 @@ # Read-only task, so don't report changed. changed_when: false with_items: "{{ tor_instances }}" + when: "v2_onion_services" tags: - tor - admin @@ -26,6 +28,42 @@ # Local action, so we don't want elevated privileges become: no with_items: "{{ tor_hidden_service_hostname_lookup.results }}" + when: "v2_onion_services" + tags: + - tor + - admin + + +- name: Wait for all Tor v3 onion services hostname files. + wait_for: + state: present + path: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}/hostname" + delay: 5 + with_items: "{{ tor_instances_v3 }}" + when: "v3_onion_services" + tags: + - tor + +- name: Collect Tor v3 onion service hostnames. + command: cat /var/lib/tor/services/{{ item.service }}/hostname + register: tor_hidden_service_hostnamev3_lookup + # Read-only task, so don't report changed. + changed_when: false + with_items: "{{ tor_instances_v3 }}" + when: "v3_onion_services" + tags: + - tor + - admin + +- name: Write Tor v3 onion service hostname files to Admin Workstation. + local_action: + module: template + dest: "{{ role_path }}/../../{{ item.item.filename }}" + src: ths_config_v3.j2 + # Local action, so we don't want elevated privileges + become: no + with_items: "{{ tor_hidden_service_hostnamev3_lookup.results }}" + when: "v3_onion_services" tags: - tor - admin diff --git a/install_files/ansible-base/roles/restrict-direct-access/tasks/main.yml b/install_files/ansible-base/roles/restrict-direct-access/tasks/main.yml index 1127edf4b3..2be2d68ca2 100644 --- a/install_files/ansible-base/roles/restrict-direct-access/tasks/main.yml +++ b/install_files/ansible-base/roles/restrict-direct-access/tasks/main.yml @@ -1,5 +1,6 @@ --- - include: fetch_tor_config.yml + when: fetch_tor_client_auth_configs - include: dh_moduli.yml diff --git a/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2 b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2 new file mode 100644 index 0000000000..9a535d462c --- /dev/null +++ b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2 @@ -0,0 +1 @@ +{% if item.item.filename.endswith('.auth_private') %}{{ item.stdout|truncate(56, True, '') }}:descriptor:x25519:{{ (lookup('file', role_path+'/../../tor_v3_keys.json')|from_json)[tor_v3_service_map[item.item.filename]] }}{% else %}{{ item.stdout }}{% endif %} diff --git a/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml b/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml index 6e86c95ddf..47359a2292 100644 --- a/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml +++ b/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml @@ -2,3 +2,10 @@ tor_hidden_services_parent_dir: /var/lib/tor/services tor_user: debian-tor enable_ssh_over_tor: true +sd_root_dir: "{{ lookup('pipe','git rev-parse --show-toplevel') }}" + +# v2 Tor onion services are on / v3 Tor onion services are off by default for backwards +# compatibility. Note that new install after 1.0 will have v3 enabled by sdconfig which +# will override these variables. +v2_onion_services: true +v3_onion_services: false diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml new file mode 100644 index 0000000000..5f62d454f5 --- /dev/null +++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml @@ -0,0 +1,39 @@ +--- +# The migration from Tor v2 -> v3 services requires Admins to opt-in +# via `securedrop-admin sdconfig`, then rerun the install action to enable +# v3 services. If a *different* Admin Workstation is subsequently used, +# without the corresponding sdconfig changes, we halt the play, otherwise +# the v3 client auth config will be clobbered, locking out the first Admin. + +- name: Check whether v3 services exist on server + stat: + path: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}" + register: _v3_services_existence_check_result + with_items: "{{ tor_instances_v3 }}" + + # Returns a list of booleans, one boolean for each v3 service, denoting whether + # a config currently exists for that service. +- name: Store info about existing v3 service state + set_fact: + _v3_services_state_info: "{{ _v3_services_existence_check_result.results|map(attribute='stat')|map(attribute='exists')|list }}" + + # Fail if v3 service configs exist, but v3_onion_services is not enabled. + # If v3 service configs do not exist, but v3_onion_services is enabled, + # we're configuring the v3 services for the first time, so don't fail. +- name: Confirm service state matches declared config + assert: + that: > + not (not v3_onion_services and true in _v3_services_state_info) + msg: > + ERROR. The 'sdconfig' settings do not specify v3 Onion Services, + but v3 Onion Services were found on the server. If your SecureDrop + instance has multiple Administrators, contact the other Administrators + to request the Tor v3 Onion Service config information. You must copy + the 'tor_v3_keys.json' and '*.auth_private' files to this workstation, + then re-run the install action. + when: + # In staging, Monitor Server will have 0 (SSH-over-Tor disabled) + - tor_instances_v3|length > 0 + # Only run if we're connected over Tor (i.e. enabling v3 after v2). + # If we're not connected over Tor, this is a first-run. + - (ansible_host|default(ansible_ssh_host)).endswith('.onion') diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml index e2daed1f26..fc3cabd8d0 100644 --- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml +++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml @@ -16,7 +16,24 @@ owner: "{{ tor_user }}" group: "{{ tor_user }}" mode: "0700" - with_items: "{{ tor_instances }}" + with_flattened: + - "{{ tor_instances if v2_onion_services else [] }}" + - "{{ tor_instances_v3 if v3_onion_services else [] }}" + tags: + - tor + +- name: Create directories for Tor v3 authenticated onion services. + file: + state: directory + dest: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}/authorized_clients" + owner: "{{ tor_user }}" + group: "{{ tor_user }}" + mode: "0700" + with_items: "{{ tor_instances_v3 }}" + when: + - v3_onion_services + # Source Interface is always public, don't configure client auth + - "'source' not in item.service" tags: - tor @@ -32,8 +49,80 @@ tags: - tor +# TODO: While cryptography is available in the development requirements, this task does +# requires the install of admin requirements in whatever virtualenv is activated for running +# staging (this seems reasonable to me). +- name: Generate Onion v3 keys if required the Tails admin system + command: "python {{ sd_root_dir }}/admin/securedrop_admin/__init__.py --root {{ sd_root_dir }} generate_v3_keys" + delegate_to: localhost + # Local action, so we don't want elevated privileges + become: no + when: "v3_onion_services" + register: onion_v3_generation + changed_when: "'onion service keys generated' in onion_v3_generation.stdout" + tags: + - tor + - admin + +- name: Get the v3 keys locally from the Tails admin system + set_fact: + v3_local_key_info: "{{ lookup('file', role_path+'/../../tor_v3_keys.json')|from_json }}" + delegate_to: localhost + # Local action, so we don't want elevated privileges + become: no + # Suppress output since it contains Tor keys + no_log: true + when: "v3_onion_services" + tags: + - tor + - admin + +- name: Look up SSH v3 pubkey info. + set_fact: + tor_v3_ssh_pubkey: "{{ v3_local_key_info.app_ssh_public_key if 'securedrop_application_server' in group_names else v3_local_key_info.mon_ssh_public_key }}" + when: + - v3_onion_services + - "'sshv3' in tor_auth_instances_v3" + - enable_ssh_over_tor + tags: + - tor + +- name: Create the client auth file for the app server for Journalist interface + copy: + dest: "{{ tor_hidden_services_parent_dir }}/journalistv3/authorized_clients/client.auth" + content: | + descriptor:x25519:{{ v3_local_key_info.app_journalist_public_key }} + owner: "{{ tor_user }}" + group: "{{ tor_user }}" + mode: "0600" + notify: + - restart tor + when: + - v3_onion_services + - "'journalistv3' in tor_auth_instances_v3" + tags: + - tor + +- name: Create the client auth file for the app server for ssh interface + copy: + dest: "{{ tor_hidden_services_parent_dir }}/sshv3/authorized_clients/client.auth" + content: | + descriptor:x25519:{{ tor_v3_ssh_pubkey }} + owner: "{{ tor_user }}" + group: "{{ tor_user }}" + mode: "0600" + notify: + - restart tor + when: + - v3_onion_services + - "'sshv3' in tor_auth_instances_v3" + - enable_ssh_over_tor + tags: + - tor + - name: Flush handlers to restart Tor. meta: flush_handlers + when: "v3_onion_services" tags: - tor @@ -41,5 +130,6 @@ service: name: tor state: started + when: "v3_onion_services" tags: - tor diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/main.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/main.yml index 3e24b6a0ff..baa1012094 100644 --- a/install_files/ansible-base/roles/tor-hidden-services/tasks/main.yml +++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/main.yml @@ -1,4 +1,6 @@ --- - include: install_tor.yml +- include: check_tor_service_config_for_admins.yml + - include: configure_tor_hidden_services.yml diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/torrc b/install_files/ansible-base/roles/tor-hidden-services/templates/torrc index 1a711b6692..d7464fda80 100644 --- a/install_files/ansible-base/roles/tor-hidden-services/templates/torrc +++ b/install_files/ansible-base/roles/tor-hidden-services/templates/torrc @@ -2,7 +2,7 @@ SocksPort 0 SafeLogging 1 RunAsDaemon 1 -{% if 'securedrop_application_server' in group_names %} +{% if 'securedrop_application_server' in group_names and v2_onion_services %} HiddenServiceDir /var/lib/tor/services/source HiddenServiceVersion 2 HiddenServicePort 80 127.0.0.1:80 @@ -22,3 +22,21 @@ HiddenServiceVersion 2 HiddenServicePort 22 127.0.0.1:22 HiddenServiceAuthorizeClient stealth admin {% endif %} + + +{% if 'securedrop_application_server' in group_names and v3_onion_services %} +HiddenServiceDir /var/lib/tor/services/sourcev3 +HiddenServicePort 80 127.0.0.1:80 + +{% if securedrop_app_https_on_source_interface|default(False) %} +HiddenServicePort 443 127.0.0.1:443 +{% endif %} + +HiddenServiceDir /var/lib/tor/services/journalistv3 +HiddenServicePort 80 127.0.0.1:8080 +{% endif %} + +{% if enable_ssh_over_tor and v3_onion_services %} +HiddenServiceDir /var/lib/tor/services/sshv3 +HiddenServicePort 22 127.0.0.1:22 +{% endif %} diff --git a/install_files/ansible-base/securedrop-prod.yml b/install_files/ansible-base/securedrop-prod.yml index 6526173f94..87b8421912 100755 --- a/install_files/ansible-base/securedrop-prod.yml +++ b/install_files/ansible-base/securedrop-prod.yml @@ -37,6 +37,9 @@ - name: Include restrict role early when using ssh over localnet include_role: name: restrict-direct-access + vars: + # Don't wait for tor client auth, might not exist yet + fetch_tor_client_auth_configs: false when: - not enable_ssh_over_tor - sd_dir_check.stat.exists diff --git a/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf b/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf index 9b14f4a818..35778f56be 100644 --- a/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf +++ b/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf @@ -14,16 +14,29 @@ /var/www /var/lib/securedrop /var/lib/tor/services/source/hostname + /var/lib/tor/services/sourcev3/hostname /var/lib/tor/services/journalist/hostname + /var/lib/tor/services/journalistv3/hostname /var/lib/tor/services/ssh/hostname + /var/lib/tor/services/sshv3/hostname /var/lib/tor/lock /boot - /var/lib/tor/services/source/private_keys + + + /var/lib/tor/services/source/private_key /var/lib/tor/services/journalist/client_keys + /var/lib/tor/services/journalist/private_key + /var/lib/tor/services/ssh/private_key /var/lib/tor/services/ssh/client_keys + /var/lib/tor/services/journalistv3/authorized_clients + /var/lib/tor/services/journalistv3/hs_ed25519_secret_key + /var/lib/tor/services/sourcev3/hs_ed25519_secret_key + /var/lib/tor/services/sshv3/authorized_clients + /var/lib/tor/services/sshv3/hs_ed25519_secret_key + /var/lib/securedrop/keys/random_seed /var/lib/securedrop/keys/pubring.gpg /var/lib/securedrop/keys/secring.gpg diff --git a/molecule/testinfra/staging/app/test_tor_hidden_services.py b/molecule/testinfra/staging/app/test_tor_hidden_services.py index 18d21c2299..319cdd1f25 100644 --- a/molecule/testinfra/staging/app/test_tor_hidden_services.py +++ b/molecule/testinfra/staging/app/test_tor_hidden_services.py @@ -29,6 +29,7 @@ def test_tor_service_hostnames(host, tor_service): # Declare regex only for THS; we'll build regex for ATHS only if # necessary, since we won't have the required values otherwise. ths_hostname_regex = r"[a-z0-9]{16}\.onion" + ths_hostname_regex_v3 = r"[a-z0-9]{56}\.onion" with host.sudo(): f = host.file("/var/lib/tor/services/{}/hostname".format( @@ -41,15 +42,24 @@ def test_tor_service_hostnames(host, tor_service): # All hostnames should contain at *least* the hostname. assert re.search(ths_hostname_regex, f.content_string) - if tor_service['authenticated']: + if tor_service['authenticated'] and tor_service['version'] == 2: # HidServAuth regex is approximately [a-zA-Z0-9/+], but validating # the entire entry is sane, and we don't need to nitpick the # charset. aths_hostname_regex = ths_hostname_regex + " .{22} # client: " + \ tor_service['client'] assert re.search("^{}$".format(aths_hostname_regex), f.content_string) - else: + elif tor_service['authenticated'] and tor_service['version'] == 3: + # For authenticated version 3 onion services, the authorized_client + # directory will exist and contain a file called client.auth. + client_auth = host.file( + "/var/lib/tor/services/{}/authorized_clients/client.auth".format( + tor_service['name'])) + assert client_auth.is_file + elif tor_service['version'] == 2: assert re.search("^{}$".format(ths_hostname_regex), f.content_string) + else: + assert re.search("^{}$".format(ths_hostname_regex_v3), f.content_string) @pytest.mark.parametrize('tor_service', sdvars.tor_services) @@ -61,7 +71,7 @@ def test_tor_services_config(host, tor_service): * HiddenServiceDir * HiddenServicePort - Only authenticated Onion Services must also include: + Only v2 authenticated Onion Services must also include: * HiddenServiceAuthorizeClient @@ -81,7 +91,10 @@ def test_tor_services_config(host, tor_service): # Ensure that service is hardcoded to v2, for compatibility # with newer versions of Tor, which default to v3. - version_string = "HiddenServiceVersion 2" + if tor_service['version'] == 2: + version_string = "HiddenServiceVersion 2" + else: + version_string = "" port_regex = "HiddenServicePort {} 127.0.0.1:{}".format( remote_port, local_port) @@ -89,9 +102,12 @@ def test_tor_services_config(host, tor_service): assert f.contains("^{}$".format(dir_regex)) assert f.contains("^{}$".format(port_regex)) - service_regex = "\n".join([dir_regex, version_string, port_regex]) + if version_string: + service_regex = "\n".join([dir_regex, version_string, port_regex]) + else: + service_regex = "\n".join([dir_regex, port_regex]) - if tor_service['authenticated']: + if tor_service['authenticated'] and tor_service['version'] == 2: auth_regex = "HiddenServiceAuthorizeClient stealth {}".format( tor_service['client']) assert f.contains("^{}$".format(auth_regex)) diff --git a/molecule/testinfra/staging/vars/app-staging.yml b/molecule/testinfra/staging/vars/app-staging.yml index ee47e2fd83..1477456136 100644 --- a/molecule/testinfra/staging/vars/app-staging.yml +++ b/molecule/testinfra/staging/vars/app-staging.yml @@ -56,6 +56,7 @@ tor_services: ports: - "80" authenticated: no + version: 2 - name: journalist ports: @@ -63,6 +64,19 @@ tor_services: - "8080" authenticated: yes client: journalist + version: 2 + + - name: journalistv3 + ports: + - "80" + authenticated: yes + version: 3 + + - name: sourcev3 + ports: + - "80" + authenticated: no + version: 3 # Staging permits presence of "source-error.log". allowed_apache_logfiles: diff --git a/molecule/testinfra/staging/vars/staging.yml b/molecule/testinfra/staging/vars/staging.yml index 2049389d6a..f18c27d8ff 100644 --- a/molecule/testinfra/staging/vars/staging.yml +++ b/molecule/testinfra/staging/vars/staging.yml @@ -56,6 +56,7 @@ tor_services: ports: - "80" authenticated: no + version: 2 - name: journalist ports: @@ -63,6 +64,19 @@ tor_services: - "8080" authenticated: yes client: journalist + version: 2 + + - name: journalistv3 + ports: + - "80" + authenticated: yes + version: 3 + + - name: sourcev3 + ports: + - "80" + authenticated: no + version: 3 # Staging permits presence of "source-error.log". allowed_apache_logfiles: diff --git a/securedrop/requirements/python2/develop-requirements.txt b/securedrop/requirements/python2/develop-requirements.txt index 6a8fc9e1ed..9830af978b 100644 --- a/securedrop/requirements/python2/develop-requirements.txt +++ b/securedrop/requirements/python2/develop-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file=requirements/python2/develop-requirements.txt ../admin/requirements-ansible.in requirements/python2/develop-requirements.in +# pip-compile --output-file=requirements/python2/develop-requirements.txt ../admin/requirements-ansible.in ../admin/requirements.in requirements/python2/develop-requirements.in # alabaster==0.7.10 # via sphinx ansible-lint==4.1.0 # via molecule @@ -32,7 +32,7 @@ click==6.7 # via click-completion, cookiecutter, git-url-parse, m colorama==0.3.9 # via molecule, python-gilt configparser==3.7.4 # via entrypoints, flake8, pylint cookiecutter==1.6.0 # via molecule -cryptography==2.3.1 +cryptography==2.7 dnspython==1.15.0 docker-py==1.10.6 docker-pycreds==0.2.1 # via docker-py @@ -51,7 +51,7 @@ git-url-parse==1.0.2 # via python-gilt gitdb2==2.0.3 # via gitpython gitpython==2.1.8 # via bandit html-linter==0.4.0 -idna==2.5 # via cryptography, molecule, requests +idna==2.5 # via molecule, requests imagesize==0.7.1 # via sphinx ipaddress==1.0.22 # via cryptography, docker-py isort==4.2.15 # via pylint @@ -74,6 +74,7 @@ pexpect==4.6.0 # via molecule pip-tools==4.0.0 port-for==0.3.1 # via sphinx-autobuild poyo==0.4.1 # via cookiecutter +prompt-toolkit==2.0.9 psutil==5.4.6 # via molecule ptyprocess==0.5.2 # via pexpect py==1.4.34 # via pytest @@ -100,7 +101,7 @@ s3transfer==0.1.12 # via boto3 safety==1.8.4 sh==1.12.14 # via molecule, python-gilt singledispatch==3.4.0.3 # via astroid, pylint, tornado -six==1.11.0 # via ansible-lint, astroid, bandit, bcrypt, click-completion, cryptography, docker-py, docker-pycreds, dparse, fasteners, git-url-parse, livereload, molecule, packaging, pip-tools, pylint, pynacl, python-dateutil, singledispatch, sphinx, stevedore, testinfra, websocket-client +six==1.11.0 # via ansible-lint, astroid, bandit, bcrypt, click-completion, cryptography, docker-py, docker-pycreds, dparse, fasteners, git-url-parse, livereload, molecule, packaging, pip-tools, prompt-toolkit, pylint, pynacl, python-dateutil, singledispatch, sphinx, stevedore, testinfra, websocket-client smmap2==2.0.3 # via gitdb2 snowballstemmer==1.2.1 # via sphinx sphinx-autobuild==0.7.1 @@ -116,6 +117,7 @@ tree-format==0.1.2 # via molecule typing==3.6.6 # via flake8, sphinx urllib3==1.25.3 watchdog==0.8.3 # via sphinx-autobuild +wcwidth==0.1.7 # via prompt-toolkit websocket-client==0.44.0 # via docker-py whichcraft==0.4.1 # via cookiecutter wrapt==1.10.11 # via astroid diff --git a/securedrop/requirements/python3/develop-requirements.txt b/securedrop/requirements/python3/develop-requirements.txt index 9471850c66..ac111022a7 100644 --- a/securedrop/requirements/python3/develop-requirements.txt +++ b/securedrop/requirements/python3/develop-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file=requirements/python3/develop-requirements.txt ../admin/requirements-ansible.in requirements/python3/develop-requirements.in +# pip-compile --output-file=requirements/python3/develop-requirements.txt ../admin/requirements-ansible.in ../admin/requirements.in requirements/python3/develop-requirements.in # alabaster==0.7.10 # via sphinx ansible-lint==4.1.0 # via molecule @@ -28,7 +28,7 @@ click-completion==0.3.1 # via molecule click==6.7 # via click-completion, cookiecutter, git-url-parse, molecule, pip-tools, python-gilt, safety colorama==0.3.9 # via molecule, python-gilt cookiecutter==1.6.0 # via molecule -cryptography==2.3.1 +cryptography==2.7 dnspython==1.15.0 docker-py==1.10.6 docker-pycreds==0.2.1 # via docker-py @@ -44,7 +44,7 @@ git-url-parse==1.0.2 # via python-gilt gitdb2==2.0.3 # via gitpython gitpython==2.1.8 # via bandit html-linter==0.4.0 -idna==2.5 # via cryptography, molecule, requests +idna==2.5 # via molecule, requests imagesize==0.7.1 # via sphinx isort==4.2.15 # via pylint jinja2-time==0.2.0 # via cookiecutter @@ -68,6 +68,7 @@ pexpect==4.6.0 # via molecule pip-tools==4.0.0 port_for==0.3.1 # via sphinx-autobuild poyo==0.4.1 # via cookiecutter +prompt-toolkit==2.0.9 psutil==5.4.6 # via molecule ptyprocess==0.5.2 # via pexpect py==1.4.34 # via pytest @@ -92,7 +93,7 @@ ruamel.yaml==0.15.97 # via ansible-lint s3transfer==0.1.12 # via boto3 safety==1.8.4 sh==1.12.14 # via molecule, python-gilt -six==1.11.0 # via ansible-lint, astroid, bandit, bcrypt, click-completion, cryptography, docker-py, docker-pycreds, dparse, fasteners, git-url-parse, livereload, molecule, packaging, pip-tools, pylint, pynacl, python-dateutil, sphinx, stevedore, testinfra, websocket-client +six==1.11.0 # via ansible-lint, astroid, bandit, bcrypt, click-completion, cryptography, docker-py, docker-pycreds, dparse, fasteners, git-url-parse, livereload, molecule, packaging, pip-tools, prompt-toolkit, pylint, pynacl, python-dateutil, sphinx, stevedore, testinfra, websocket-client smmap2==2.0.3 # via gitdb2 snowballstemmer==1.2.1 # via sphinx sphinx-autobuild==0.7.1 @@ -108,6 +109,7 @@ tree-format==0.1.2 # via molecule typed-ast==1.3.5 # via mypy urllib3==1.25.3 watchdog==0.8.3 # via sphinx-autobuild +wcwidth==0.1.7 # via prompt-toolkit websocket-client==0.44.0 # via docker-py whichcraft==0.4.1 # via cookiecutter wrapt==1.10.11 # via astroid