From 039861a667c093ab56b30f3e5f0fe5b50ef2628b Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Tue, 1 Sep 2020 18:23:47 +0530 Subject: [PATCH] Adds patched dh-virtualenv to build package on xenial This patched deployment.py file for dh-virtualenv installs known working setuptools==46.0.0 in the virtualenv, and then it will try to install dependencies for SecureDrop. Fixes #5471 --- .../files/deployment.py | 249 ++++++++++++++++++ .../files/setup-requirements.txt | 3 + .../tasks/main.yml | 11 +- 3 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/deployment.py create mode 100644 install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/setup-requirements.txt diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/deployment.py b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/deployment.py new file mode 100644 index 0000000000..c93088e47b --- /dev/null +++ b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/deployment.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2013 - 2014 Spotify AB + +# This file is part of dh-virtualenv. + +# dh-virtualenv is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 2 of the +# License, or (at your option) any later version. + +# dh-virtualenv is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with dh-virtualenv. If not, see +# . + +import os +import re +import shutil +import subprocess +import tempfile + +ROOT_ENV_KEY = 'DH_VIRTUALENV_INSTALL_ROOT' +DEFAULT_INSTALL_DIR = '/usr/share/python/' + + +class Deployment(object): + def __init__(self, package, extra_urls=[], preinstall=[], + pypi_url=None, setuptools=False, python=None, + builtin_venv=False, sourcedirectory=None, verbose=False, + extra_pip_arg=[], use_system_packages=False, + skip_install=False, + install_suffix=None, + requirements_filename='requirements.txt'): + + self.package = package + install_root = os.environ.get(ROOT_ENV_KEY, DEFAULT_INSTALL_DIR) + self.install_suffix = install_suffix + + self.debian_root = os.path.join( + 'debian', package, install_root.lstrip('/')) + + if install_suffix is None: + self.virtualenv_install_dir = os.path.join(install_root, self.package) + self.package_dir = os.path.join(self.debian_root, package) + else: + self.virtualenv_install_dir = os.path.join(install_root, install_suffix) + self.package_dir = os.path.join(self.debian_root, install_suffix) + + self.bin_dir = os.path.join(self.package_dir, 'bin') + self.local_bin_dir = os.path.join(self.package_dir, 'local', 'bin') + + self.extra_urls = extra_urls + self.preinstall = preinstall + self.extra_pip_arg = extra_pip_arg + self.pypi_url = pypi_url + self.log_file = tempfile.NamedTemporaryFile() + self.verbose = verbose + self.setuptools = setuptools + self.python = python + self.builtin_venv = builtin_venv + self.sourcedirectory = '.' if sourcedirectory is None else sourcedirectory + self.use_system_packages = use_system_packages + self.skip_install = skip_install + self.requirements_filename = requirements_filename + + @classmethod + def from_options(cls, package, options): + verbose = options.verbose or os.environ.get('DH_VERBOSE') == '1' + return cls(package, + extra_urls=options.extra_index_url, + preinstall=options.preinstall, + pypi_url=options.pypi_url, + setuptools=options.setuptools, + python=options.python, + builtin_venv=options.builtin_venv, + sourcedirectory=options.sourcedirectory, + verbose=verbose, + extra_pip_arg=options.extra_pip_arg, + use_system_packages=options.use_system_packages, + skip_install=options.skip_install, + install_suffix=options.install_suffix, + requirements_filename=options.requirements_filename) + + def clean(self): + shutil.rmtree(self.debian_root) + + def create_virtualenv(self): + if self.builtin_venv: + virtualenv = [self.python, '-m', 'venv'] + else: + virtualenv = ['virtualenv'] + + if self.use_system_packages: + virtualenv.append('--system-site-packages') + else: + virtualenv.append('--no-site-packages') + + if self.setuptools: + virtualenv.append('--setuptools') + + if self.verbose: + virtualenv.append('--verbose') + + if self.python: + virtualenv.extend(('--python', self.python)) + + virtualenv.append(self.package_dir) + subprocess.check_call(virtualenv) + + # We need to prefix the pip run with the location of python + # executable. Otherwise it would just blow up due to too long + # shebang-line. + self.pip_prefix = [ + os.path.abspath(os.path.join(self.bin_dir, 'python')), + os.path.abspath(os.path.join(self.bin_dir, 'pip')), + ] + + if self.verbose: + self.pip_prefix.append('-v') + + self.pip_prefix.append('install') + + if self.pypi_url: + self.pip_prefix.append('--pypi-url={0}'.format(self.pypi_url)) + self.pip_prefix.extend([ + '--extra-index-url={0}'.format(url) for url in self.extra_urls + ]) + self.pip_prefix.append('--log={0}'.format(os.path.abspath(self.log_file.name))) + # Add in any user supplied pip args + if self.extra_pip_arg: + self.pip_prefix.extend(self.extra_pip_arg) + + def pip(self, *args): + return self.pip_prefix + list(args) + + def install_dependencies(self): + # Install preinstall stage packages. This is handy if you need + # a custom package to install dependencies (think something + # along lines of setuptools), but that does not get installed + # by default virtualenv. + if self.preinstall: + subprocess.check_call(self.pip(*self.preinstall)) + self.pip_prefix2 = [ + os.path.abspath(os.path.join(self.bin_dir, 'python')), + os.path.abspath(os.path.join(self.bin_dir, 'pip')), + "install", "--no-cache-dir", "--require-hashes", "-r", + "/tmp/setup-requirements.txt"] # nosec + + subprocess.check_call(self.pip_prefix2) + requirements_path = os.path.join(self.sourcedirectory, self.requirements_filename) + if os.path.exists(requirements_path): + subprocess.check_call(self.pip('-r', requirements_path)) + + def run_tests(self): + python = os.path.abspath(os.path.join(self.bin_dir, 'python')) + setup_py = os.path.join(self.sourcedirectory, 'setup.py') + if os.path.exists(setup_py): + subprocess.check_call([python, 'setup.py', 'test'], cwd=self.sourcedirectory) + + def fix_shebangs(self): + """Translate /usr/bin/python and /usr/bin/env python sheband + lines to point to our virtualenv python. + """ + grep_proc = subprocess.Popen( + ['grep', '-l', '-r', '-e', r'^#!.*bin/\(env \)\?python', + self.bin_dir], + stdout=subprocess.PIPE + ) + files, stderr = grep_proc.communicate() + files = files.strip() + if not files: + return + + pythonpath = os.path.join(self.virtualenv_install_dir, 'bin/python') + for f in files.split('\n'): + subprocess.check_call( + ['sed', '-i', r's|^#!.*bin/\(env \)\?python|#!{0}|'.format( + pythonpath), + f]) + + def fix_activate_path(self): + """Replace the `VIRTUAL_ENV` path in bin/activate to reflect the + post-install path of the virtualenv. + """ + activate_settings = [ + [ + 'VIRTUAL_ENV="{0}"'.format(self.virtualenv_install_dir), + r'^VIRTUAL_ENV=.*$', + "activate" + ], + [ + 'setenv VIRTUAL_ENV "{0}"'.format(self.virtualenv_install_dir), + r'^setenv VIRTUAL_ENV.*$', + "activate.csh" + ], + [ + 'set -gx VIRTUAL_ENV "{0}"'.format(self.virtualenv_install_dir), + r'^set -gx VIRTUAL_ENV.*$', + "activate.fish" + ], + ] + + for activate_args in activate_settings: + virtualenv_path = activate_args[0] + pattern = re.compile(activate_args[1], flags=re.M) + activate_file = activate_args[2] + + with open(os.path.join(self.bin_dir, activate_file), 'r+') as fh: + content = pattern.sub(virtualenv_path, fh.read()) + fh.seek(0) + fh.truncate() + fh.write(content) + + def install_package(self): + if not self.skip_install: + subprocess.check_call(self.pip('.'), cwd=os.path.abspath(self.sourcedirectory)) + + def fix_local_symlinks(self): + # The virtualenv might end up with a local folder that points outside the package + # Specifically it might point at the build environment that created it! + # Make those links relative + # See https://github.com/pypa/virtualenv/commit/5cb7cd652953441a6696c15bdac3c4f9746dfaa1 + local_dir = os.path.join(self.package_dir, "local") + if not os.path.isdir(local_dir): + return + elif os.path.samefile(self.package_dir, local_dir): + # "local" points directly to its containing directory + os.unlink(local_dir) + os.symlink(".", local_dir) + return + + for d in os.listdir(local_dir): + path = os.path.join(local_dir, d) + if not os.path.islink(path): + continue + + existing_target = os.readlink(path) + if not os.path.isabs(existing_target): + # If the symlink is already relative, we don't + # want to touch it. + continue + + new_target = os.path.relpath(existing_target, local_dir) + os.unlink(path) + os.symlink(new_target, path) diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/setup-requirements.txt b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/setup-requirements.txt new file mode 100644 index 0000000000..4af834c60d --- /dev/null +++ b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/setup-requirements.txt @@ -0,0 +1,3 @@ +setuptools==46.0.0 \ + --hash=sha256:2f00f25b780fbfd0787e46891dcccd805b08d007621f24629025f48afef444b5 \ + --hash=sha256:693e0504490ed8420522bf6bc3aa4b0da6a9f1c80c68acfb4e959275fd04cd82 diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/main.yml b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/main.yml index 268b0c87f6..e55d333d14 100644 --- a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/main.yml +++ b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/main.yml @@ -100,13 +100,22 @@ - "{{ securedrop_app_code_deb_dir }}/var/www/securedrop/static/.webassets-cache" - "{{ securedrop_app_code_deb_dir }}/var/www/securedrop/.well-known/pki-validation" +- name: Copy deployment.py file + copy: + src: "deployment.py" + dest: "/usr/lib/python2.7/dist-packages/dh_virtualenv/deployment.py" + +- name: Copy setuptools requirements file + copy: + src: "setup-requirements.txt" + dest: "/tmp/setup-requirements.txt" + - name: Build securedrop-app-code Debian package command: dpkg-buildpackage -us -uc args: chdir: "{{ securedrop_app_code_deb_dir }}" environment: DH_VIRTUALENV_INSTALL_ROOT: "/opt/venvs" - SETUPTOOLS_USE_DISTUTILS: "stdlib" - name: Find newly built Debian package find: