From d2262f4c9e8f778394ff3be4827cf6cb20ae84d8 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Mon, 14 Aug 2017 20:02:08 +0100 Subject: [PATCH 01/16] Fixes idempotence issue when installing packages with versions ending in ".0". --- conda.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/conda.py b/conda.py index c7f126c..e8be628 100755 --- a/conda.py +++ b/conda.py @@ -107,6 +107,16 @@ def _add_extras_to_command(command, extras): return command +def _are_versions_equal(version_a, version_b): + """ + Checks whether two versions are equal. + :param version_a: the first version + :param version_b: the first version + :return: `True` if versions are equal + """ + return version_a.rstrip(".0") == version_b.rstrip(".0") + + def _check_installed(module, conda, name): """ Check whether a package is installed. Returns (bool, version_str). @@ -127,7 +137,7 @@ def _check_installed(module, conda, name): installed = False version = None - + data = json.loads(stdout) if data: # At this point data will be a list of len 1, with the element of @@ -186,7 +196,7 @@ def _install_package( the latest version if no version is specified. """ - if installed and (version is None or installed_version == version): + if installed and (version is None or _are_versions_equal(installed_version, version)): module.exit_json(changed=False, name=name, version=version) if module.check_mode: From 84b2ec587164cab0cd4f43200110fa11758c3b31 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Fri, 25 Aug 2017 14:17:02 +0100 Subject: [PATCH 02/16] Adds regression test. --- tests/tasks/main.yml | 5 ++ .../test-install-loosely-fixed-version.yml | 54 +++++++++++++++++++ tests/vars/main.yml | 1 + 3 files changed, 60 insertions(+) create mode 100644 tests/tasks/test-install-loosely-fixed-version.yml diff --git a/tests/tasks/main.yml b/tests/tasks/main.yml index cf1b4d6..82a1307 100644 --- a/tests/tasks/main.yml +++ b/tests/tasks/main.yml @@ -13,6 +13,11 @@ always: - include: tear-down.yml +- block: + - include: test-install-loosely-fixed-version.yml + always: + - include: tear-down.yml + - block: - include: test-upgrade.yml always: diff --git a/tests/tasks/test-install-loosely-fixed-version.yml b/tests/tasks/test-install-loosely-fixed-version.yml new file mode 100644 index 0000000..d5c710d --- /dev/null +++ b/tests/tasks/test-install-loosely-fixed-version.yml @@ -0,0 +1,54 @@ +--- + +- name: install conda package + conda: + name: "{{ conda_tests_install_example }}" + version: "{{ conda_tests_minimum_latest_major_version }}" + state: present + executable: "{{ conda_tests_conda_executable }}" + register: first_install + +- include: set-install-facts.yml + +- name: verify installed + assert: + that: first_install.changed + that: example_package.installed + that: example_package.version | version_compare(conda_tests_minimum_latest_version, '>=') + +- name: install conda package (again) + conda: + name: "{{ conda_tests_install_example }}" + version: "{{ conda_tests_minimum_latest_major_version }}" + state: present + executable: "{{ conda_tests_conda_executable }}" + register: second_install + +- name: verify idempotence + assert: + that: not second_install.changed + +- name: install older conda package + conda: + name: "{{ conda_tests_install_example }}" + version: "{{ conda_tests_old_version }}" + state: present + executable: "{{ conda_tests_conda_executable }}" + +- name: upgrade conda package + conda: + name: "{{ conda_tests_install_example }}" + version: "{{ conda_tests_minimum_latest_major_version }}" + state: present + executable: "{{ conda_tests_conda_executable }}" + register: upgrade_install + +- include: set-install-facts.yml + +- name: verify upgrade + assert: + that: upgrade_install.changed + that: example_package.installed + that: example_package.version | version_compare(conda_tests_minimum_latest_version, '>=') + + diff --git a/tests/vars/main.yml b/tests/vars/main.yml index 2db4d07..5569927 100644 --- a/tests/vars/main.yml +++ b/tests/vars/main.yml @@ -3,4 +3,5 @@ # XXX: better to hit local package repository with dummy package conda_tests_install_example: translationstring conda_tests_minimum_latest_version: "1.3" +conda_tests_minimum_latest_major_version: "1" conda_tests_old_version: "1.1" From 271e329d2591437611a96a63493fb1a30fac17fe Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Tue, 5 Sep 2017 15:47:29 +0100 Subject: [PATCH 03/16] Refactored and reworked module to fix subtle version matching issues. --- conda.py | 268 +++++++++--------- tests/tasks/main.yml | 5 + .../test-install-loosely-fixed-version.yml | 5 + 3 files changed, 142 insertions(+), 136 deletions(-) diff --git a/conda.py b/conda.py index e8be628..60f41cb 100755 --- a/conda.py +++ b/conda.py @@ -56,6 +56,7 @@ from distutils.spawn import find_executable import os.path import json +from ansible.module_utils.basic import AnsibleModule def _find_conda(module, executable): @@ -63,7 +64,6 @@ def _find_conda(module, executable): If `executable` is not None, checks whether it points to a valid file and returns it if this is the case. Otherwise tries to find the `conda` executable in the path. Calls `fail_json` if either of these fail. - """ if not executable: conda = find_executable('conda') @@ -80,7 +80,6 @@ def _add_channels_to_command(command, channels): """ Add extra channels to a conda command by splitting the channels and putting "--channel" before each one. - """ if channels: channels = channels.strip().split() @@ -98,7 +97,6 @@ def _add_extras_to_command(command, extras): """ Add extra arguments to a conda command by splitting the arguments on white space and inserting them after the second item in the command. - """ if extras: extras = extras.strip().split() @@ -107,173 +105,168 @@ def _add_extras_to_command(command, extras): return command -def _are_versions_equal(version_a, version_b): +def _parse_stdout_as_json(stdout): """ - Checks whether two versions are equal. - :param version_a: the first version - :param version_b: the first version - :return: `True` if versions are equal + Parses the given output from stdout as JSON. + :param stdout: the output from stdout + :return: standard out as parsed JSON else `None` if non-JSON format """ - return version_a.rstrip(".0") == version_b.rstrip(".0") + try: + return json.loads(stdout) + except ValueError: + return None -def _check_installed(module, conda, name): +def _run_conda_command(module, command): """ - Check whether a package is installed. Returns (bool, version_str). - + Runs the given Conda related command. + + It is assumed that the command will return JSON. + :param module: the Ansible module + :param command: the command to run + :return: tuple where the first element is the parsed JSON output returned by Conda and the second is what was + written to standard error + :raises CondaCommandError: if there a problem running Conda """ - command = [ - conda, - 'list', - '^' + name + '$', - '--json' - ] + command = _add_channels_to_command(command, module.params['channels']) command = _add_extras_to_command(command, module.params['extra_args']) rc, stdout, stderr = module.run_command(command) + parsed_stdout = _parse_stdout_as_json(stdout) - if rc != 0: - return False, None + if rc != 0 or parsed_stdout is None: + error_message = None + if parsed_stdout is not None and 'message' in parsed_stdout: + error_message = parsed_stdout['message'] + raise CondaCommandError(command, error_message, parsed_stdout, stdout, stderr) - installed = False - version = None + return parsed_stdout, stderr - data = json.loads(stdout) - if data: - # At this point data will be a list of len 1, with the element of - # the format: "channel::package-version-py35_1" - line = data[0] - if "::" in line: - channel, other = line.split('::') - else: - other = line - - if isinstance(other, dict): - pname = other.get('name', '') - pversion = other.get('version', '') - else: - # split carefully as some package names have "-" in them (scikit-learn) - pname, pversion, pdist = other.rsplit('-', 2) - - if pname == name: # verify match for safety - installed = True - version = pversion - return installed, version +def _run_conda_package_command(module, name, version, command): + """ + Runs a Conda command related to a particular package. + :param module: the Ansible module + :param name: the name of the package the command refers to + :param version: the version of the package that the command is referring to + :param command: the Conda command + :raises CondaPackageNotFoundError: if the package referred to by this command is not found + """ + try: + return _run_conda_command(module, command) + except CondaCommandError as e: + if e.output is not None and 'exception_name' in e.output \ + and e.output['exception_name'] == 'PackageNotFoundError': + raise CondaPackageNotFoundError(name, version) + else: + raise -def _remove_package(module, conda, installed, name): +def _get_install_target(name, version): """ - Use conda to remove a given package if it is installed. - + Gets install target string for a package with the given name and version. + :param name: the package name + :param version: the package version (`None` if latest) + :return: the target string that Conda can refer to the given package as """ - if module.check_mode and installed: - module.exit_json(changed=True) - - if not installed: - module.exit_json(changed=False) - - command = [ - conda, - 'remove', - '--yes', - name - ] - command = _add_extras_to_command(command, module.params['extra_args']) + install_target = name + if version is not None: + install_target = '%s=%s' % (name, version) + return install_target - rc, stdout, stderr = module.run_command(command) - if rc != 0: - module.fail_json(msg='failed to remove package ' + name, stderr=stderr) +def _check_package_installed(module, conda, name, version): + """ + Check whether a package with the given name and version is installed. + :param module: the Ansible module + :param name: the name of the package to check if installed + :param version: the version of the package to check if installed (`None` if check for latest) + :return: `True` if a package with the given name and version is installed + :raises CondaUnexpectedOutputError: if the JSON returned by Conda was unexpected + """ + output, stderr = _run_conda_package_command( + module, name, version, [conda, 'install', '--json', '--dry-run', _get_install_target(name, version)]) - module.exit_json(changed=True, name=name, stdout=stdout, stderr=stderr) + if 'message' in output and output['message'] == 'All requested packages already installed.': + return True + elif 'actions' in output and len(output['actions']) > 0: + return False + else: + raise CondaUnexpectedOutputError(output, stderr) -def _install_package( - module, conda, installed, name, version, installed_version): +def _install_package(module, conda, name, version=None): """ - Install a package at a specific version, or install a missing package at - the latest version if no version is specified. - + Install a package with the given name and version. Version will default to latest if `None`. """ - if installed and (version is None or _are_versions_equal(installed_version, version)): - module.exit_json(changed=False, name=name, version=version) - + command = [conda, 'install', '--yes', '--json', _get_install_target(name, version)] if module.check_mode: - if not installed or (installed and installed_version != version): - module.exit_json(changed=True) + command.insert(-1, '--dry-run') - if version: - install_str = name + '=' + version - else: - install_str = name - - command = [ - conda, - 'install', - '--yes', - install_str - ] - command = _add_channels_to_command(command, module.params['channels']) - command = _add_extras_to_command(command, module.params['extra_args']) + output, stderr = _run_conda_package_command(module, name, version, command) + module.exit_json(changed=True, name=name, version=version, output=output, error=stderr) - rc, stdout, stderr = module.run_command(command) - if rc != 0: - module.fail_json(msg='failed to install package ' + name, stderr=stderr) +def _uninstall_package(module, conda, name): + """ + Use Conda to remove a package with the given name. + """ + command = [conda, 'remove', '--yes', '--json', name] + if module.check_mode: + command.insert(-1, '--dry-run') - module.exit_json( - changed=True, name=name, version=version, stdout=stdout, stderr=stderr) + output, stderr = _run_conda_package_command(module, name, None, command) + module.exit_json(changed=True, output=output, error=stderr) -def _update_package(module, conda, installed, name): +class CondaCommandError(Exception): """ - Make sure an installed package is at its latest version. - + Error raised when a Conda command fails. """ - if not installed: - module.fail_json(msg='can\'t update a package that is not installed') - - # see if it's already installed at the latest version - command = [ - conda, - 'update', - '--dry-run', - name - ] - command = _add_channels_to_command(command, module.params['channels']) - command = _add_extras_to_command(command, module.params['extra_args']) + def __init__(self, command, error_message, output, stdout, stderr): + self.command = command + self.error_message = error_message + self.output = output + self.stdout = stdout + self.stderr = stderr - rc, stdout, stderr = module.run_command(command) + def __str__(self): + error_message = ' Error: %s.' % self.error_message if self.error_message is not None else '' + stdout = ' stdout: %s.' % self.stdout if self.error_message is None and self.stdout.strip() != '' else '' + stderr = ' stderr: %s.' % self.stderr if self.stderr.strip() != '' else '' + return 'Error running command: %s.%s%s%s' % (self.command, error_message, stdout, stderr) - if rc != 0: - module.fail_json(msg='can\'t update a package that is not installed', stderr=stderr) - if 'requested packages already installed' in stdout: - module.exit_json(changed=False, name=name) +class CondaPackageNotFoundError(Exception): + """ + Error raised when a Conda package has not been found in the package repositories that were searched. + """ + def __int__(self, name, version): + self.name = name + self.version = version - # now we're definitely gonna update the package - if module.check_mode: - module.exit_json(changed=True, name=name) - - command = [ - conda, - 'update', - '--yes', - name - ] - command = _add_channels_to_command(command, module.params['channels']) - command = _add_extras_to_command(command, module.params['extra_args']) + def __str__(self): + return 'Conda package "%s" not found' % (_get_install_target(self.name, self.version)) - rc, stdout, stderr = module.run_command(command) - if rc != 0: - module.fail_json(msg='failed to update package ' + name, stderr=stderr) +class CondaUnexpectedOutputError(Exception): + """ + Error raised when the running of a Conda command has resulted in an unexpected output. + """ + def __int__(self, output, stderr): + self.output = output + self.stderr = stderr - module.exit_json(changed=True, name=name, stdout=stdout, stderr=stderr) + def __str__(self): + stderr = 'stderr: %s' % self.stderr if self.stderr.strip() != '' else '' + return 'Unexpected output from Conda (may be due to a change in Conda\'s output format): "%output".%s' \ + % (self.output, stderr) def main(): + """ + Entrypoint. + """ module = AnsibleModule( argument_spec={ 'name': {'required': True, 'type': 'str'}, @@ -294,19 +287,22 @@ def main(): state = module.params['state'] version = module.params['version'] - installed, installed_version = _check_installed(module, conda, name) + if state == 'latest' and version != None: + module.fail_json(msg='`version` must not be set if `state == "latest"` (`latest` upgrades to newest version)') + + correct_version_installed = _check_package_installed(module, conda, name, version) + + if not correct_version_installed and state != 'absent': + _install_package(module, conda, name, version) if state == 'absent': - _remove_package(module, conda, installed, name) - elif state == 'present' or (state == 'latest' and not installed): - _install_package( - module, conda, installed, name, version, installed_version) - elif state == 'latest': - _update_package(module, conda, installed, name) + try: + _uninstall_package(module, conda, name) + except CondaPackageNotFoundError: + pass + module.exit_json(changed=False) -# import module snippets -from ansible.module_utils.basic import * if __name__ == '__main__': main() diff --git a/tests/tasks/main.yml b/tests/tasks/main.yml index 82a1307..8c5b60e 100644 --- a/tests/tasks/main.yml +++ b/tests/tasks/main.yml @@ -3,6 +3,11 @@ - include: install.yml - include: tear-down.yml +- block: + - include: test-uninstall.yml + always: + - include: tear-down.yml + - block: - include: test-install-latest.yml always: diff --git a/tests/tasks/test-install-loosely-fixed-version.yml b/tests/tasks/test-install-loosely-fixed-version.yml index d5c710d..8b8dcbb 100644 --- a/tests/tasks/test-install-loosely-fixed-version.yml +++ b/tests/tasks/test-install-loosely-fixed-version.yml @@ -28,6 +28,11 @@ assert: that: not second_install.changed + +- name: verify setup for testing package upgrade + assert: + that: conda_tests_old_version.split('.')[0] == conda_tests_minimum_latest_major_version + - name: install older conda package conda: name: "{{ conda_tests_install_example }}" From c42f05d5207adb92f9bf5c27de32ea85de4c5bd7 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Tue, 5 Sep 2017 15:49:22 +0100 Subject: [PATCH 04/16] Adds regression tests for uninstalling Conda packages. --- tests/tasks/test-uninstall.yml | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/tasks/test-uninstall.yml diff --git a/tests/tasks/test-uninstall.yml b/tests/tasks/test-uninstall.yml new file mode 100644 index 0000000..99a57a3 --- /dev/null +++ b/tests/tasks/test-uninstall.yml @@ -0,0 +1,35 @@ +--- + +- name: install Conda package + conda: + name: "{{ conda_tests_install_example }}" + state: latest + executable: "{{ conda_tests_conda_executable }}" + register: first_install + +- name: uninstall Conda package + conda: + name: "{{ conda_tests_install_example }}" + state: absent + executable: "{{ conda_tests_conda_executable }}" + register: first_uninstall + +- include: set-install-facts.yml + +- name: verify package not installed + assert: + that: first_uninstall.changed + that: not example_package.installed + +- name: uninstall Conda package again + conda: + name: "{{ conda_tests_install_example }}" + state: absent + executable: "{{ conda_tests_conda_executable }}" + register: second_uninstall + +- include: set-install-facts.yml + +- name: verify idempotence + assert: + that: not second_uninstall.changed From b92e1cba98dbc6bbd7eed61b34aa0ba13d786c1c Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Tue, 5 Sep 2017 16:52:45 +0100 Subject: [PATCH 05/16] Improvement naming in tests. --- tests/tasks/set-install-facts.yml | 2 +- tests/tasks/test-downgrade.yml | 4 ++-- tests/tasks/test-install-fixed-version.yml | 4 ++-- tests/tasks/test-install-latest.yml | 6 ++++-- tests/tasks/test-install-loosely-fixed-version.yml | 8 ++++---- tests/tasks/test-upgrade.yml | 4 ++-- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/tasks/set-install-facts.yml b/tests/tasks/set-install-facts.yml index b51284f..b55faeb 100644 --- a/tests/tasks/set-install-facts.yml +++ b/tests/tasks/set-install-facts.yml @@ -1,6 +1,6 @@ --- -- name: find installed Conda matching packages +- name: find installed matching Conda packages shell: "{{ conda_tests_conda_executable }} search --json ^{{ conda_tests_install_example }}$" no_log: True register: installed_raw diff --git a/tests/tasks/test-downgrade.yml b/tests/tasks/test-downgrade.yml index 3cb0f81..dd75e12 100644 --- a/tests/tasks/test-downgrade.yml +++ b/tests/tasks/test-downgrade.yml @@ -1,12 +1,12 @@ --- -- name: install latest package via conda +- name: install latest Conda package conda: name: "{{ conda_tests_install_example }}" state: latest executable: "{{ conda_tests_conda_executable }}" -- name: install old package via conda +- name: install old Conda package conda: name: "{{ conda_tests_install_example }}" version: "{{ conda_tests_old_version }}" diff --git a/tests/tasks/test-install-fixed-version.yml b/tests/tasks/test-install-fixed-version.yml index 1755970..3edaf4f 100644 --- a/tests/tasks/test-install-fixed-version.yml +++ b/tests/tasks/test-install-fixed-version.yml @@ -1,6 +1,6 @@ --- -- name: install package via conda +- name: install Conda package conda: name: "{{ conda_tests_install_example }}" version: "{{ conda_tests_old_version }}" @@ -16,7 +16,7 @@ that: example_package.installed that: example_package.version | version_compare(conda_tests_old_version, '=') -- name: install package via conda (again) +- name: install Conda package (again) conda: name: "{{ conda_tests_install_example }}" version: "{{ conda_tests_old_version }}" diff --git a/tests/tasks/test-install-latest.yml b/tests/tasks/test-install-latest.yml index dd93e10..ed76a83 100644 --- a/tests/tasks/test-install-latest.yml +++ b/tests/tasks/test-install-latest.yml @@ -1,6 +1,6 @@ --- -- name: install package via conda +- name: install latest version of the Conda package conda: name: "{{ conda_tests_install_example }}" state: latest @@ -15,7 +15,9 @@ that: example_package.installed that: example_package.version | version_compare(conda_tests_minimum_latest_version, '>=') -- name: install package via conda (again) +# TODO: Until tests are ran against local package repository, this test would be flaky in the (rare) case that the +# package is updated mid-test +- name: install latest version of the Conda package (again) conda: name: "{{ conda_tests_install_example }}" state: latest diff --git a/tests/tasks/test-install-loosely-fixed-version.yml b/tests/tasks/test-install-loosely-fixed-version.yml index 8b8dcbb..207eff7 100644 --- a/tests/tasks/test-install-loosely-fixed-version.yml +++ b/tests/tasks/test-install-loosely-fixed-version.yml @@ -1,6 +1,6 @@ --- -- name: install conda package +- name: install Conda package using major version number conda: name: "{{ conda_tests_install_example }}" version: "{{ conda_tests_minimum_latest_major_version }}" @@ -16,7 +16,7 @@ that: example_package.installed that: example_package.version | version_compare(conda_tests_minimum_latest_version, '>=') -- name: install conda package (again) +- name: install Conda package using major version number (again) conda: name: "{{ conda_tests_install_example }}" version: "{{ conda_tests_minimum_latest_major_version }}" @@ -33,14 +33,14 @@ assert: that: conda_tests_old_version.split('.')[0] == conda_tests_minimum_latest_major_version -- name: install older conda package +- name: install older Conda package conda: name: "{{ conda_tests_install_example }}" version: "{{ conda_tests_old_version }}" state: present executable: "{{ conda_tests_conda_executable }}" -- name: upgrade conda package +- name: upgrade Conda package conda: name: "{{ conda_tests_install_example }}" version: "{{ conda_tests_minimum_latest_major_version }}" diff --git a/tests/tasks/test-upgrade.yml b/tests/tasks/test-upgrade.yml index 011853c..9c11613 100644 --- a/tests/tasks/test-upgrade.yml +++ b/tests/tasks/test-upgrade.yml @@ -1,13 +1,13 @@ --- -- name: install old package via conda +- name: install old Conda package conda: name: "{{ conda_tests_install_example }}" version: "{{ conda_tests_old_version }}" state: present executable: "{{ conda_tests_conda_executable }}" -- name: install latest package via conda +- name: install latest Conda package conda: name: "{{ conda_tests_install_example }}" state: latest From 8bb1b6dd349af4a8e1faaa0f7d6d10a33ad3e4d9 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Tue, 5 Sep 2017 16:54:27 +0100 Subject: [PATCH 06/16] Adds regression tests for invalid setups. --- tests/tasks/test-invalid-setups.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/tasks/test-invalid-setups.yml diff --git a/tests/tasks/test-invalid-setups.yml b/tests/tasks/test-invalid-setups.yml new file mode 100644 index 0000000..4e37556 --- /dev/null +++ b/tests/tasks/test-invalid-setups.yml @@ -0,0 +1,14 @@ +--- + +- name: install Conda packge with fixed version but `latest` state + conda: + name: "{{ conda_tests_install_example }}" + version: "{{ conda_tests_old_version }}" + state: latest + executable: "{{ conda_tests_conda_executable }}" + register: versioned_and_latest_install + ignore_errors: yes + +- name: verify invalid setup + assert: + that: versioned_and_latest_install.failed From 42eb6306417c27c5cda2f5952b5ad9613146d165 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Tue, 5 Sep 2017 16:59:26 +0100 Subject: [PATCH 07/16] Places Ansible integration tests into separate subdirectory. --- tests/{ => integration}/defaults/main.yml | 0 tests/integration/library/conda.py | 1 + tests/{ => integration}/meta/main.yml | 0 tests/{ => integration}/requirements.txt | 0 tests/{ => integration}/requirements.yml | 0 tests/{ => integration}/site.retry | 0 tests/{ => integration}/site.yml | 0 tests/{ => integration}/tasks/install.yml | 0 tests/{ => integration}/tasks/main.yml | 5 +++++ tests/{ => integration}/tasks/set-install-facts.yml | 0 tests/{ => integration}/tasks/tear-down.yml | 0 tests/{ => integration}/tasks/test-downgrade.yml | 0 tests/{ => integration}/tasks/test-install-fixed-version.yml | 0 tests/{ => integration}/tasks/test-install-latest.yml | 0 .../tasks/test-install-loosely-fixed-version.yml | 0 tests/{ => integration}/tasks/test-invalid-setups.yml | 0 tests/{ => integration}/tasks/test-uninstall.yml | 0 tests/{ => integration}/tasks/test-upgrade.yml | 0 tests/{ => integration}/vars/main.yml | 0 tests/library/conda.py | 1 - 20 files changed, 6 insertions(+), 1 deletion(-) rename tests/{ => integration}/defaults/main.yml (100%) create mode 120000 tests/integration/library/conda.py rename tests/{ => integration}/meta/main.yml (100%) rename tests/{ => integration}/requirements.txt (100%) rename tests/{ => integration}/requirements.yml (100%) rename tests/{ => integration}/site.retry (100%) rename tests/{ => integration}/site.yml (100%) rename tests/{ => integration}/tasks/install.yml (100%) rename tests/{ => integration}/tasks/main.yml (86%) rename tests/{ => integration}/tasks/set-install-facts.yml (100%) rename tests/{ => integration}/tasks/tear-down.yml (100%) rename tests/{ => integration}/tasks/test-downgrade.yml (100%) rename tests/{ => integration}/tasks/test-install-fixed-version.yml (100%) rename tests/{ => integration}/tasks/test-install-latest.yml (100%) rename tests/{ => integration}/tasks/test-install-loosely-fixed-version.yml (100%) rename tests/{ => integration}/tasks/test-invalid-setups.yml (100%) rename tests/{ => integration}/tasks/test-uninstall.yml (100%) rename tests/{ => integration}/tasks/test-upgrade.yml (100%) rename tests/{ => integration}/vars/main.yml (100%) delete mode 120000 tests/library/conda.py diff --git a/tests/defaults/main.yml b/tests/integration/defaults/main.yml similarity index 100% rename from tests/defaults/main.yml rename to tests/integration/defaults/main.yml diff --git a/tests/integration/library/conda.py b/tests/integration/library/conda.py new file mode 120000 index 0000000..d69432a --- /dev/null +++ b/tests/integration/library/conda.py @@ -0,0 +1 @@ +../../../conda.py \ No newline at end of file diff --git a/tests/meta/main.yml b/tests/integration/meta/main.yml similarity index 100% rename from tests/meta/main.yml rename to tests/integration/meta/main.yml diff --git a/tests/requirements.txt b/tests/integration/requirements.txt similarity index 100% rename from tests/requirements.txt rename to tests/integration/requirements.txt diff --git a/tests/requirements.yml b/tests/integration/requirements.yml similarity index 100% rename from tests/requirements.yml rename to tests/integration/requirements.yml diff --git a/tests/site.retry b/tests/integration/site.retry similarity index 100% rename from tests/site.retry rename to tests/integration/site.retry diff --git a/tests/site.yml b/tests/integration/site.yml similarity index 100% rename from tests/site.yml rename to tests/integration/site.yml diff --git a/tests/tasks/install.yml b/tests/integration/tasks/install.yml similarity index 100% rename from tests/tasks/install.yml rename to tests/integration/tasks/install.yml diff --git a/tests/tasks/main.yml b/tests/integration/tasks/main.yml similarity index 86% rename from tests/tasks/main.yml rename to tests/integration/tasks/main.yml index 8c5b60e..cc90d69 100644 --- a/tests/tasks/main.yml +++ b/tests/integration/tasks/main.yml @@ -32,3 +32,8 @@ - include: test-downgrade.yml always: - include: tear-down.yml + +- block: + - include: test-invalid-setups.yml + always: + - include: tear-down.yml diff --git a/tests/tasks/set-install-facts.yml b/tests/integration/tasks/set-install-facts.yml similarity index 100% rename from tests/tasks/set-install-facts.yml rename to tests/integration/tasks/set-install-facts.yml diff --git a/tests/tasks/tear-down.yml b/tests/integration/tasks/tear-down.yml similarity index 100% rename from tests/tasks/tear-down.yml rename to tests/integration/tasks/tear-down.yml diff --git a/tests/tasks/test-downgrade.yml b/tests/integration/tasks/test-downgrade.yml similarity index 100% rename from tests/tasks/test-downgrade.yml rename to tests/integration/tasks/test-downgrade.yml diff --git a/tests/tasks/test-install-fixed-version.yml b/tests/integration/tasks/test-install-fixed-version.yml similarity index 100% rename from tests/tasks/test-install-fixed-version.yml rename to tests/integration/tasks/test-install-fixed-version.yml diff --git a/tests/tasks/test-install-latest.yml b/tests/integration/tasks/test-install-latest.yml similarity index 100% rename from tests/tasks/test-install-latest.yml rename to tests/integration/tasks/test-install-latest.yml diff --git a/tests/tasks/test-install-loosely-fixed-version.yml b/tests/integration/tasks/test-install-loosely-fixed-version.yml similarity index 100% rename from tests/tasks/test-install-loosely-fixed-version.yml rename to tests/integration/tasks/test-install-loosely-fixed-version.yml diff --git a/tests/tasks/test-invalid-setups.yml b/tests/integration/tasks/test-invalid-setups.yml similarity index 100% rename from tests/tasks/test-invalid-setups.yml rename to tests/integration/tasks/test-invalid-setups.yml diff --git a/tests/tasks/test-uninstall.yml b/tests/integration/tasks/test-uninstall.yml similarity index 100% rename from tests/tasks/test-uninstall.yml rename to tests/integration/tasks/test-uninstall.yml diff --git a/tests/tasks/test-upgrade.yml b/tests/integration/tasks/test-upgrade.yml similarity index 100% rename from tests/tasks/test-upgrade.yml rename to tests/integration/tasks/test-upgrade.yml diff --git a/tests/vars/main.yml b/tests/integration/vars/main.yml similarity index 100% rename from tests/vars/main.yml rename to tests/integration/vars/main.yml diff --git a/tests/library/conda.py b/tests/library/conda.py deleted file mode 120000 index 797b559..0000000 --- a/tests/library/conda.py +++ /dev/null @@ -1 +0,0 @@ -../../conda.py \ No newline at end of file From 7157b8c87c92ed3e8a5e7e0f9057aacc0226c751 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Wed, 6 Sep 2017 10:59:10 +0100 Subject: [PATCH 08/16] Handles Conda spewing progess messages onto stdout. --- conda.py | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/conda.py b/conda.py index 60f41cb..ffd1e66 100755 --- a/conda.py +++ b/conda.py @@ -76,7 +76,7 @@ def _find_conda(module, executable): module.fail_json(msg="could not find conda executable") -def _add_channels_to_command(command, channels): +def add_channels_to_command(command, channels): """ Add extra channels to a conda command by splitting the channels and putting "--channel" before each one. @@ -93,7 +93,7 @@ def _add_channels_to_command(command, channels): return command -def _add_extras_to_command(command, extras): +def add_extras_to_command(command, extras): """ Add extra arguments to a conda command by splitting the arguments on white space and inserting them after the second item in the command. @@ -105,14 +105,27 @@ def _add_extras_to_command(command, extras): return command -def _parse_stdout_as_json(stdout): +def parse_conda_stdout(stdout): """ - Parses the given output from stdout as JSON. + Parses the given output from Conda. :param stdout: the output from stdout :return: standard out as parsed JSON else `None` if non-JSON format """ + # Conda spews loading progress reports onto stdout(!?), which need ignoring. Bug observed in Conda version 4.3.25. + split_lines = stdout.strip().split("\n") + while len(split_lines) > 0: + line = split_lines.pop(0) + try: + line_content = json.loads(line) + if "progress" not in line_content and "maxval" not in line_content: + # Looks like this was the output, not a progress update + return line_content + except ValueError: + split_lines.insert(0, line) + break + try: - return json.loads(stdout) + return json.loads("".join(split_lines)) except ValueError: return None @@ -128,11 +141,11 @@ def _run_conda_command(module, command): written to standard error :raises CondaCommandError: if there a problem running Conda """ - command = _add_channels_to_command(command, module.params['channels']) - command = _add_extras_to_command(command, module.params['extra_args']) + command = add_channels_to_command(command, module.params['channels']) + command = add_extras_to_command(command, module.params['extra_args']) rc, stdout, stderr = module.run_command(command) - parsed_stdout = _parse_stdout_as_json(stdout) + parsed_stdout = parse_conda_stdout(stdout) if rc != 0 or parsed_stdout is None: error_message = None @@ -162,7 +175,7 @@ def _run_conda_package_command(module, name, version, command): raise -def _get_install_target(name, version): +def get_install_target(name, version): """ Gets install target string for a package with the given name and version. :param name: the package name @@ -185,7 +198,7 @@ def _check_package_installed(module, conda, name, version): :raises CondaUnexpectedOutputError: if the JSON returned by Conda was unexpected """ output, stderr = _run_conda_package_command( - module, name, version, [conda, 'install', '--json', '--dry-run', _get_install_target(name, version)]) + module, name, version, [conda, 'install', '--json', '--dry-run', get_install_target(name, version)]) if 'message' in output and output['message'] == 'All requested packages already installed.': return True @@ -199,7 +212,7 @@ def _install_package(module, conda, name, version=None): """ Install a package with the given name and version. Version will default to latest if `None`. """ - command = [conda, 'install', '--yes', '--json', _get_install_target(name, version)] + command = [conda, 'install', '--yes', '--json', get_install_target(name, version)] if module.check_mode: command.insert(-1, '--dry-run') @@ -246,7 +259,7 @@ def __int__(self, name, version): self.version = version def __str__(self): - return 'Conda package "%s" not found' % (_get_install_target(self.name, self.version)) + return 'Conda package "%s" not found' % (get_install_target(self.name, self.version)) class CondaUnexpectedOutputError(Exception): @@ -287,7 +300,7 @@ def main(): state = module.params['state'] version = module.params['version'] - if state == 'latest' and version != None: + if state == 'latest' and version is not None: module.fail_json(msg='`version` must not be set if `state == "latest"` (`latest` upgrades to newest version)') correct_version_installed = _check_package_installed(module, conda, name, version) @@ -299,7 +312,7 @@ def main(): try: _uninstall_package(module, conda, name) except CondaPackageNotFoundError: - pass + """ EAFP """ module.exit_json(changed=False) From 793cf1be3087ed204865c9e1de1b8f91a6c53d15 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Thu, 7 Sep 2017 16:58:57 +0100 Subject: [PATCH 09/16] Improvements to conda.py, including making it more testable. --- conda.py | 228 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 127 insertions(+), 101 deletions(-) diff --git a/conda.py b/conda.py index ffd1e66..716c61a 100755 --- a/conda.py +++ b/conda.py @@ -59,7 +59,86 @@ from ansible.module_utils.basic import AnsibleModule -def _find_conda(module, executable): +def run_package_operation(conda, name, version, state, dry_run, command_runner, on_failure, on_success): + """ + Runs Conda package operation. + + This method is intentionally decoupled from `AnsibleModule` to allow it to be easily tested in isolation. + :param conda: location of the Conda executable + :param name: name of the package of interest + :param version: version of the package (`None` for latest) + :param state: state the package should be in + :param dry_run: will "pretend" to make changes only if `True` + :param command_runner: method that executes a given Conda command (given as list of string arguments), which returns + JSON and returns a tuple where the first argument is the outputted JSON and the second is anything written to stderr + :param on_failure: method that takes any kwargs to be called on failure + :param on_success: method that takes any kwargs to be called on success + """ + correct_version_installed = check_package_installed(command_runner, conda, name, version) + + # TODO: State should be an "enum" (or whatever the Py2.7 equivalent is) + if not correct_version_installed and state != 'absent': + try: + output, stderr = install_package(command_runner, conda, name, version, dry_run=dry_run) + on_success(changed=True, output=output, error=stderr) + except CondaPackageNotFoundError: + on_failure(msg='Conda package "%s" not found' % (get_install_target(name, version, ))) + + elif state == 'absent': + try: + output, stderr = uninstall_package(command_runner, conda, name, dry_run=dry_run) + on_success(changed=True, output=output, error=stderr) + except CondaPackageNotFoundError: + on_success(changed=False) + + else: + on_success(changed=False) + + +def check_package_installed(command_runner, conda, name, version): + """ + Check whether a package with the given name and version is installed. + :param command_runner: method that executes a given Conda command (given as list of string arguments), which returns + JSON and returns a tuple where the first argument is the outputted JSON and the second is anything written to stderr + :param name: the name of the package to check if installed + :param version: the version of the package to check if installed (`None` if check for latest) + :return: `True` if a package with the given name and version is installed + :raises CondaUnexpectedOutputError: if the JSON returned by Conda was unexpected + """ + output, stderr = run_conda_package_command( + command_runner, name, version, [conda, 'install', '--json', '--dry-run', get_install_target(name, version)]) + + if 'message' in output and output['message'] == 'All requested packages already installed.': + return True + elif 'actions' in output and len(output['actions']) > 0: + return False + else: + raise CondaUnexpectedOutputError(output, stderr) + + +def install_package(command_runner, conda, name, version=None, dry_run=False): + """ + Install a package with the given name and version. Version will default to latest if `None`. + """ + command = [conda, 'install', '--yes', '--json', get_install_target(name, version)] + if dry_run: + command.insert(-1, '--dry-run') + + return run_conda_package_command(command_runner, name, version, command) + + +def uninstall_package(command_runner, conda, name, dry_run=False): + """ + Use Conda to remove a package with the given name. + """ + command = [conda, 'remove', '--yes', '--json', name] + if dry_run: + command.insert(-1, '--dry-run') + + return run_conda_package_command(command_runner, name, None, command) + + +def find_conda(executable): """ If `executable` is not None, checks whether it points to a valid file and returns it if this is the case. Otherwise tries to find the `conda` @@ -73,7 +152,7 @@ def _find_conda(module, executable): if os.path.isfile(executable): return executable - module.fail_json(msg="could not find conda executable") + raise CondaExecutableNotFoundError() def add_channels_to_command(command, channels): @@ -130,43 +209,17 @@ def parse_conda_stdout(stdout): return None -def _run_conda_command(module, command): - """ - Runs the given Conda related command. - - It is assumed that the command will return JSON. - :param module: the Ansible module - :param command: the command to run - :return: tuple where the first element is the parsed JSON output returned by Conda and the second is what was - written to standard error - :raises CondaCommandError: if there a problem running Conda - """ - command = add_channels_to_command(command, module.params['channels']) - command = add_extras_to_command(command, module.params['extra_args']) - - rc, stdout, stderr = module.run_command(command) - parsed_stdout = parse_conda_stdout(stdout) - - if rc != 0 or parsed_stdout is None: - error_message = None - if parsed_stdout is not None and 'message' in parsed_stdout: - error_message = parsed_stdout['message'] - raise CondaCommandError(command, error_message, parsed_stdout, stdout, stderr) - - return parsed_stdout, stderr - - -def _run_conda_package_command(module, name, version, command): +def run_conda_package_command(command_runner, name, version, command): """ Runs a Conda command related to a particular package. - :param module: the Ansible module + :param command_runner: runner of Conda commands :param name: the name of the package the command refers to :param version: the version of the package that the command is referring to :param command: the Conda command :raises CondaPackageNotFoundError: if the package referred to by this command is not found """ try: - return _run_conda_command(module, command) + return command_runner(command) except CondaCommandError as e: if e.output is not None and 'exception_name' in e.output \ and e.output['exception_name'] == 'PackageNotFoundError': @@ -188,66 +241,21 @@ def get_install_target(name, version): return install_target -def _check_package_installed(module, conda, name, version): - """ - Check whether a package with the given name and version is installed. - :param module: the Ansible module - :param name: the name of the package to check if installed - :param version: the version of the package to check if installed (`None` if check for latest) - :return: `True` if a package with the given name and version is installed - :raises CondaUnexpectedOutputError: if the JSON returned by Conda was unexpected - """ - output, stderr = _run_conda_package_command( - module, name, version, [conda, 'install', '--json', '--dry-run', get_install_target(name, version)]) - - if 'message' in output and output['message'] == 'All requested packages already installed.': - return True - elif 'actions' in output and len(output['actions']) > 0: - return False - else: - raise CondaUnexpectedOutputError(output, stderr) - - -def _install_package(module, conda, name, version=None): - """ - Install a package with the given name and version. Version will default to latest if `None`. - """ - command = [conda, 'install', '--yes', '--json', get_install_target(name, version)] - if module.check_mode: - command.insert(-1, '--dry-run') - - output, stderr = _run_conda_package_command(module, name, version, command) - module.exit_json(changed=True, name=name, version=version, output=output, error=stderr) - - -def _uninstall_package(module, conda, name): - """ - Use Conda to remove a package with the given name. - """ - command = [conda, 'remove', '--yes', '--json', name] - if module.check_mode: - command.insert(-1, '--dry-run') - - output, stderr = _run_conda_package_command(module, name, None, command) - module.exit_json(changed=True, output=output, error=stderr) - - class CondaCommandError(Exception): """ Error raised when a Conda command fails. """ - def __init__(self, command, error_message, output, stdout, stderr): + def __init__(self, command, output, stdout, stderr): self.command = command - self.error_message = error_message self.output = output self.stdout = stdout self.stderr = stderr - def __str__(self): - error_message = ' Error: %s.' % self.error_message if self.error_message is not None else '' - stdout = ' stdout: %s.' % self.stdout if self.error_message is None and self.stdout.strip() != '' else '' + error_message = ' Error: %s.' % self.output['message'] if output is not None and 'message' in output else '' + stdout = ' stdout: %s.' % self.stdout if error_message is '' and self.stdout.strip() != '' else '' stderr = ' stderr: %s.' % self.stderr if self.stderr.strip() != '' else '' - return 'Error running command: %s.%s%s%s' % (self.command, error_message, stdout, stderr) + super(CondaCommandError, self).__init__( + 'Error running command: %s.%s%s%s' % (self.command, error_message, stdout, stderr)) class CondaPackageNotFoundError(Exception): @@ -257,9 +265,8 @@ class CondaPackageNotFoundError(Exception): def __int__(self, name, version): self.name = name self.version = version - - def __str__(self): - return 'Conda package "%s" not found' % (get_install_target(self.name, self.version)) + super(CondaPackageNotFoundError, self).__init__( + 'Conda package "%s" not found' % (get_install_target(self.name, self.version), )) class CondaUnexpectedOutputError(Exception): @@ -270,13 +277,39 @@ def __int__(self, output, stderr): self.output = output self.stderr = stderr - def __str__(self): stderr = 'stderr: %s' % self.stderr if self.stderr.strip() != '' else '' - return 'Unexpected output from Conda (may be due to a change in Conda\'s output format): "%output".%s' \ - % (self.output, stderr) + super(CondaUnexpectedOutputError, self).__init__( + 'Unexpected output from Conda (may be due to a change in Conda\'s output format): "%output".%s' + % (self.output, stderr)) + + +class CondaExecutableNotFoundError(Exception): + """ + Error raised when the Conda executable was not found. + """ + def __init__(self): + super(CondaExecutableNotFoundError, self).__init__('Conda executable not found.') + + +def _run_conda_command(module, command): + """ + Runs the given Conda command. + :param module: Ansible module + :param command: the Conda command to run, which must return JSON + """ + command = add_channels_to_command(command, module.params['channels']) + command = add_extras_to_command(command, module.params['extra_args']) + + rc, stdout, stderr = module.run_command(command) + output = parse_conda_stdout(stdout) + + if rc != 0 or output is None: + raise CondaCommandError(command, output, stdout, stderr) + return output, stderr -def main(): + +def _main(): """ Entrypoint. """ @@ -295,7 +328,7 @@ def main(): }, supports_check_mode=True) - conda = _find_conda(module, module.params['executable']) + conda = find_conda(module.params['executable']) name = module.params['name'] state = module.params['state'] version = module.params['version'] @@ -303,19 +336,12 @@ def main(): if state == 'latest' and version is not None: module.fail_json(msg='`version` must not be set if `state == "latest"` (`latest` upgrades to newest version)') - correct_version_installed = _check_package_installed(module, conda, name, version) - - if not correct_version_installed and state != 'absent': - _install_package(module, conda, name, version) - - if state == 'absent': - try: - _uninstall_package(module, conda, name) - except CondaPackageNotFoundError: - """ EAFP """ + def command_runner(command): + return _run_conda_command(module, command) - module.exit_json(changed=False) + run_package_operation( + conda, name, version, state, module.check_mode, command_runner, module.fail_json, module.exit_json) if __name__ == '__main__': - main() + _main() From cec4d64bb9efe6a826090f890734119d09b0f371 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Fri, 8 Sep 2017 11:30:05 +0100 Subject: [PATCH 10/16] Fixes annoying null character issue when progress JSON is spewed into stdout. --- conda.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/conda.py b/conda.py index 716c61a..7e72f83 100755 --- a/conda.py +++ b/conda.py @@ -193,7 +193,7 @@ def parse_conda_stdout(stdout): # Conda spews loading progress reports onto stdout(!?), which need ignoring. Bug observed in Conda version 4.3.25. split_lines = stdout.strip().split("\n") while len(split_lines) > 0: - line = split_lines.pop(0) + line = split_lines.pop(0).strip('\x00') try: line_content = json.loads(line) if "progress" not in line_content and "maxval" not in line_content: @@ -220,9 +220,8 @@ def run_conda_package_command(command_runner, name, version, command): """ try: return command_runner(command) - except CondaCommandError as e: - if e.output is not None and 'exception_name' in e.output \ - and e.output['exception_name'] == 'PackageNotFoundError': + except CondaCommandJsonDescribedError as e: + if 'exception_name' in e.output and e.output['exception_name'] == 'PackageNotFoundError': raise CondaPackageNotFoundError(name, version) else: raise @@ -245,17 +244,25 @@ class CondaCommandError(Exception): """ Error raised when a Conda command fails. """ - def __init__(self, command, output, stdout, stderr): + def __init__(self, command, stdout, stderr): self.command = command - self.output = output self.stdout = stdout self.stderr = stderr - error_message = ' Error: %s.' % self.output['message'] if output is not None and 'message' in output else '' - stdout = ' stdout: %s.' % self.stdout if error_message is '' and self.stdout.strip() != '' else '' + stdout = ' stdout: %s.' % self.stdout if self.stdout.strip() != '' else '' stderr = ' stderr: %s.' % self.stderr if self.stderr.strip() != '' else '' + super(CondaCommandError, self).__init__( - 'Error running command: %s.%s%s%s' % (self.command, error_message, stdout, stderr)) + 'Error running command: %s.%s%s' % (self.command, stdout, stderr)) + + +class CondaCommandJsonDescribedError(CondaCommandError): + """ + Error raised when a Conda command does not output JSON. + """ + def __init__(self, command, output, stderr): + self.output = output + super(CondaCommandJsonDescribedError, self).__init__(command, json.dumps(output), stderr) class CondaPackageNotFoundError(Exception): @@ -303,8 +310,10 @@ def _run_conda_command(module, command): rc, stdout, stderr = module.run_command(command) output = parse_conda_stdout(stdout) - if rc != 0 or output is None: - raise CondaCommandError(command, output, stdout, stderr) + if output is None: + raise CondaCommandError(command, stdout, stderr) + if rc != 0: + raise CondaCommandJsonDescribedError(command, output, stderr) return output, stderr From 98ccf35c64e349021f8edcf01d0a602f12d018f0 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Fri, 8 Sep 2017 11:45:55 +0100 Subject: [PATCH 11/16] Adds unit tests. --- Dockerfile.test | 9 ++++----- run-tests.sh | 13 +++++++++++++ tests/unit/test_conda.py | 33 +++++++++++++++++++++++++++++++++ tests/unit/test_conda.pyc | Bin 0 -> 1580 bytes 4 files changed, 50 insertions(+), 5 deletions(-) create mode 100755 run-tests.sh create mode 100644 tests/unit/test_conda.py create mode 100644 tests/unit/test_conda.pyc diff --git a/Dockerfile.test b/Dockerfile.test index 0ae10e8..42eec75 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -15,14 +15,13 @@ RUN apt-get update \ RUN pip install setuptools wheel RUN pip install ansible==2.3.2.0 -ADD tests/requirements.yml /tmp/requirements.yml +ADD tests/integration/requirements.yml /tmp/requirements.yml RUN ansible-galaxy install -r /tmp/requirements.yml -ADD tests/requirements.txt /tmp/requirements.txt +ADD tests/integration/requirements.txt /tmp/requirements.txt RUN pip install -r /tmp/requirements.txt VOLUME "${DATA_DIRECTORY}" +WORKDIR "${DATA_DIRECTORY}" -CMD ansible-galaxy install -r "${DATA_DIRECTORY}/tests/requirements.yml" \ - && pip install -r "${DATA_DIRECTORY}/tests/requirements.txt" \ - && ansible-playbook -vvv -e ansible_python_interpreter=$(which python) -c local "${DATA_DIRECTORY}/tests/site.yml" +CMD "${DATA_DIRECTORY}/run-tests.sh" diff --git a/run-tests.sh b/run-tests.sh new file mode 100755 index 0000000..1f68fe5 --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euf -o pipefail + +# Setup +ansible-galaxy install -r "${DATA_DIRECTORY}/tests/integration/requirements.yml" +pip install -r "${DATA_DIRECTORY}/tests/integration/requirements.txt" + +# Run unit tests +PYTHONPATH=. python -m unittest discover -v -s tests/unit + +# Run integration tests +ansible-playbook -vvv -e ansible_python_interpreter=$(which python) -c local "${DATA_DIRECTORY}/tests/integration/site.yml" diff --git a/tests/unit/test_conda.py b/tests/unit/test_conda.py new file mode 100644 index 0000000..8bf19f3 --- /dev/null +++ b/tests/unit/test_conda.py @@ -0,0 +1,33 @@ +import json +import unittest + +from conda import parse_conda_stdout + + +class TestParseCondaStdout(unittest.TestCase): + """ + Tests for `parse_conda_stdout`. + """ + _VALID_STDOUT = """ + { + "actions": {}, + "success": true + } + """ + + def test_parses_invalid_stdout(self): + self.assertIsNone(parse_conda_stdout("fail")) + + def test_parses_valid_stdout(self): + self.assertEqual( + json.loads(TestParseCondaStdout._VALID_STDOUT), parse_conda_stdout(TestParseCondaStdout._VALID_STDOUT)) + + def test_parses_valid_stdout_with_progress_reports(self): + stdout = '{"maxval": 17685, "finished": false, "fetch": "translationstr", "progress": 0}\n\x00' \ + '{"maxval": 17685, "finished": true, "fetch": "translationstr", "progress": 17685}\n\x00%s' \ + % (TestParseCondaStdout._VALID_STDOUT,) + self.assertEqual(json.loads(TestParseCondaStdout._VALID_STDOUT), parse_conda_stdout(stdout)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_conda.pyc b/tests/unit/test_conda.pyc new file mode 100644 index 0000000000000000000000000000000000000000..034458415afa7d85ba3fe22d4b555a60021f4fb4 GIT binary patch literal 1580 zcmbtT+iuf95S_IXmxfeAh+BEGA|ZlQ8hD@}gb>j3PzeHZ0#g0J*0nd(!Lfs9!$obL z%CGPP{2CtsGvg$!gb+}y>|}OkclMmQxZhVAF#fg^!|ZeE{}C;GLn5J{f(#&B=n6t5 zb_X&CMlOUdsKf=MU|53z`D*Y5;0RzRtb_3&slz)5%1}-$u|wq&w%7LPiS!43-L@jt zo+mMmN;Eny@+8t|lVW1+1v2*y+L!FwWKT~V3(S1^2`%i9FoaeRh7N-#c!|S^!Ws}v zR_Zg1p$9>)UFVB8NHl!<_#N~IMd|PUiez_P&fxR;X})nFHr9N<6~#6!a%|oA52kDW zNe@oq*dTjsIWY^*RJ52G@Z#)m(!$Fm4grRg6(XbHNPdb6(rY{9>)IO0L6l~)wN2VZ zh^DmL*e!As&>pI+@}60|K=m!4C=v&gz21&;jdmj}R4GKh}>N>K~@X{JYW<_AIFto2Tz5 zQ6^l6SmbmMASQvSt_04X}C01l7cz;%2B1@uqx}EzmyffQh5DkWtI zmI-sf!yGKJIKzS#NNY+VEPJbQK4neTDZuY)qCci~U(ZSF(u|AJ;wnGM?}Boj71d^J zma%D_M Date: Fri, 8 Sep 2017 11:47:20 +0100 Subject: [PATCH 12/16] Adds gitignore file. --- .gitignore | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..183abe2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,104 @@ +# Created by .ignore support plugin (hsz.mobi) +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +### Ansible template +*.retry + From 7efcbeb1c493406b5d3cbfea623302831ef84687 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Fri, 8 Sep 2017 12:32:34 +0100 Subject: [PATCH 13/16] Adds test to install larger package with dependencies. --- .../integration/tasks/test-install-latest.yml | 22 +++++++++++++++++++ tests/integration/vars/main.yml | 2 ++ 2 files changed, 24 insertions(+) diff --git a/tests/integration/tasks/test-install-latest.yml b/tests/integration/tasks/test-install-latest.yml index ed76a83..2056b42 100644 --- a/tests/integration/tasks/test-install-latest.yml +++ b/tests/integration/tasks/test-install-latest.yml @@ -27,3 +27,25 @@ - name: verify idempotence assert: that: not second_install.changed + +- block: + - name: install larger Conda package with dependencies + conda: + name: "{{ conda_tests_larger_install_example }}" + state: present + executable: "{{ conda_tests_conda_executable }}" + register: larger_install + + - include: set-install-facts.yml + vars: + conda_tests_install_example: "{{ conda_tests_larger_install_example }}" + + - name: verify installed + assert: + that: larger_install.changed + that: example_package.installed + + always: + - include: tear-down.yml + vars: + conda_tests_install_example: "{{ conda_tests_larger_install_example }}" \ No newline at end of file diff --git a/tests/integration/vars/main.yml b/tests/integration/vars/main.yml index 5569927..3591995 100644 --- a/tests/integration/vars/main.yml +++ b/tests/integration/vars/main.yml @@ -5,3 +5,5 @@ conda_tests_install_example: translationstring conda_tests_minimum_latest_version: "1.3" conda_tests_minimum_latest_major_version: "1" conda_tests_old_version: "1.1" + +conda_tests_larger_install_example: curl From 138f78b328a3ed8a84ed2fca708c074da6be9bfb Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Fri, 8 Sep 2017 12:34:01 +0100 Subject: [PATCH 14/16] Adds tests for failure scenarios. --- tests/integration/tasks/main.yml | 5 +++ tests/integration/tasks/test-failures.yml | 39 +++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/integration/tasks/test-failures.yml diff --git a/tests/integration/tasks/main.yml b/tests/integration/tasks/main.yml index cc90d69..a448baf 100644 --- a/tests/integration/tasks/main.yml +++ b/tests/integration/tasks/main.yml @@ -37,3 +37,8 @@ - include: test-invalid-setups.yml always: - include: tear-down.yml + +- block: + - include: test-failures.yml + always: + - include: tear-down.yml diff --git a/tests/integration/tasks/test-failures.yml b/tests/integration/tasks/test-failures.yml new file mode 100644 index 0000000..12f8ec5 --- /dev/null +++ b/tests/integration/tasks/test-failures.yml @@ -0,0 +1,39 @@ +--- + +- name: install Conda packge that does not exist (expect failure) + conda: + name: this_packge_hopefully_does_not_exist + state: present + executable: "{{ conda_tests_conda_executable }}" + register: non_existent_install + ignore_errors: yes + +- name: verify failure + assert: + that: non_existent_install.failed + +- name: install Conda packge version that does not exist (expect failure) + conda: + name: "{{ conda_tests_install_example }}" + state: present + version: 9999 + executable: "{{ conda_tests_conda_executable }}" + register: non_existent_install + ignore_errors: yes + +- name: verify failure + assert: + that: non_existent_install.failed + +- name: install latest Conda packge, fixed at a specific version (expect failure) + conda: + name: "{{ conda_tests_install_example }}" + state: latest + version: "{{ conda_tests_minimum_latest_version }}" + executable: "{{ conda_tests_conda_executable }}" + register: invalid_setup + ignore_errors: yes + +- name: verify failure + assert: + that: invalid_setup.failed From cc69e1d1eee5dbd79466eb97fc598bac6e690081 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Fri, 8 Sep 2017 12:50:53 +0100 Subject: [PATCH 15/16] Updates documentation. --- README.md | 2 +- conda.py | 53 +++++++++++++++++++++++++++++++++++------------------ 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index e5bd502..961a4ea 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Options (= is mandatory): - extra_args Extra arguments passed to conda [Default: None] -= name +- name The name of a Python library to install [Default: None] - state diff --git a/conda.py b/conda.py index 7e72f83..dc40ad2 100755 --- a/conda.py +++ b/conda.py @@ -9,50 +9,67 @@ > Manage Python libraries via conda. Can install, update, and remove packages. -author: Synthicity +author: + - Synthicity + - Colin Nolan (@colin-nolan) notes: > Requires conda to already be installed. - Will look under the home directory for a conda executable. options: name: - description: The name of a Python library to install + description: The name of a Python package to install. required: true - default: null version: - description: A specific version of a library to install + description: The specific version of a package to install. required: false - default: null state: - description: State in which to leave the Python package + description: State in which to leave the Python package. "present" will install a package of the specified version + if it is not installed (will not upgrade to latest if version is unspecified - will only install + latest); "latest" will both install and subsequently upgrade a package to the latest version on each + run; "absent" will uninstall the package if installed. required: false default: present choices: [ "present", "absent", "latest" ] channels: - description: Extra channels to use when installing packages + description: Extra channels to use when installing packages. required: false - default: null executable: - description: Full path to the conda executable + description: Full path to the conda executable. required: false - default: null extra_args: - description: Extra arguments passed to conda + description: Extra arguments passed to conda. required: false - default: null """ EXAMPLES = """ - name: install numpy via conda - conda: name=numpy state=latest + conda: + name: numpy + state: latest - name: install scipy 0.14 via conda - conda: name=scipy version="0.14" + conda: + name: scipy + version: "0.14" - name: remove matplotlib from conda - conda: name=matplotlib state=absent + conda: + name: matplotlib + state: absent """ +RETURN = """ +output: + description: JSON output from Conda + returned: `changed == True` + type: dict +stderr: + description: stderr content written by Conda + returned: `changed == True` + type: str +""" + + from distutils.spawn import find_executable import os.path import json @@ -80,14 +97,14 @@ def run_package_operation(conda, name, version, state, dry_run, command_runner, if not correct_version_installed and state != 'absent': try: output, stderr = install_package(command_runner, conda, name, version, dry_run=dry_run) - on_success(changed=True, output=output, error=stderr) + on_success(changed=True, output=output, stderr=stderr) except CondaPackageNotFoundError: on_failure(msg='Conda package "%s" not found' % (get_install_target(name, version, ))) elif state == 'absent': try: output, stderr = uninstall_package(command_runner, conda, name, dry_run=dry_run) - on_success(changed=True, output=output, error=stderr) + on_success(changed=True, output=output, stderr=stderr) except CondaPackageNotFoundError: on_success(changed=False) From 178b16d897514cca1ed0ea1fff8f28965b4b6e80 Mon Sep 17 00:00:00 2001 From: Colin Nolan Date: Fri, 8 Sep 2017 14:58:53 +0100 Subject: [PATCH 16/16] Removes temp files that crept in before the gitignore was added. --- tests/integration/site.retry | 1 - tests/unit/test_conda.pyc | Bin 1580 -> 0 bytes 2 files changed, 1 deletion(-) delete mode 100644 tests/integration/site.retry delete mode 100644 tests/unit/test_conda.pyc diff --git a/tests/integration/site.retry b/tests/integration/site.retry deleted file mode 100644 index 2fbb50c..0000000 --- a/tests/integration/site.retry +++ /dev/null @@ -1 +0,0 @@ -localhost diff --git a/tests/unit/test_conda.pyc b/tests/unit/test_conda.pyc deleted file mode 100644 index 034458415afa7d85ba3fe22d4b555a60021f4fb4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1580 zcmbtT+iuf95S_IXmxfeAh+BEGA|ZlQ8hD@}gb>j3PzeHZ0#g0J*0nd(!Lfs9!$obL z%CGPP{2CtsGvg$!gb+}y>|}OkclMmQxZhVAF#fg^!|ZeE{}C;GLn5J{f(#&B=n6t5 zb_X&CMlOUdsKf=MU|53z`D*Y5;0RzRtb_3&slz)5%1}-$u|wq&w%7LPiS!43-L@jt zo+mMmN;Eny@+8t|lVW1+1v2*y+L!FwWKT~V3(S1^2`%i9FoaeRh7N-#c!|S^!Ws}v zR_Zg1p$9>)UFVB8NHl!<_#N~IMd|PUiez_P&fxR;X})nFHr9N<6~#6!a%|oA52kDW zNe@oq*dTjsIWY^*RJ52G@Z#)m(!$Fm4grRg6(XbHNPdb6(rY{9>)IO0L6l~)wN2VZ zh^DmL*e!As&>pI+@}60|K=m!4C=v&gz21&;jdmj}R4GKh}>N>K~@X{JYW<_AIFto2Tz5 zQ6^l6SmbmMASQvSt_04X}C01l7cz;%2B1@uqx}EzmyffQh5DkWtI zmI-sf!yGKJIKzS#NNY+VEPJbQK4neTDZuY)qCci~U(ZSF(u|AJ;wnGM?}Boj71d^J zma%D_M