From 263861d13f449d8a180ca8135fa1caea2ab66fa4 Mon Sep 17 00:00:00 2001 From: Guilhem Saurel Date: Sat, 7 Oct 2017 20:16:55 +0200 Subject: [PATCH 1/2] support URLs as packages (Squashed commits by @nim65s and @toejough, rebased by @jcushman) Co-Authored-By: toejough --- piptools/repositories/pypi.py | 9 +++++---- piptools/resolver.py | 13 +++++++------ piptools/sync.py | 4 ++-- piptools/utils.py | 19 ++++++++++++++++++- tests/test_cli_compile.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 13 deletions(-) diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index 3a332c1c5..c5070e168 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -28,6 +28,7 @@ from ..utils import ( fs_str, is_pinned_requirement, + is_url_requirement, lookup_table, make_install_requirement, ) @@ -136,7 +137,7 @@ def find_best_match(self, ireq, prereleases=None): Returns a Version object that indicates the best match for the given InstallRequirement according to the external repository. """ - if ireq.editable: + if ireq.editable or is_url_requirement(ireq, self): return ireq # return itself as the best match all_candidates = self.find_all_candidates(ireq.name) @@ -228,13 +229,13 @@ def resolve_reqs(self, download_dir, ireq, wheel_cache): def get_dependencies(self, ireq): """ - Given a pinned or an editable InstallRequirement, returns a set of + Given a pinned, a url, or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). They indicate the secondary dependencies for the given requirement. """ - if not (ireq.editable or is_pinned_requirement(ireq)): + if not (ireq.editable or is_url_requirement(ireq, self) or is_pinned_requirement(ireq)): raise TypeError( - "Expected pinned or editable InstallRequirement, got {}".format(ireq) + "Expected url, pinned or editable InstallRequirement, got {}".format(ireq) ) if ireq not in self._dependencies_cache: diff --git a/piptools/resolver.py b/piptools/resolver.py index 764cb6b2f..c4fdf4f60 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -17,6 +17,7 @@ format_specifier, full_groupby, is_pinned_requirement, + is_url_requirement, key_from_ireq, key_from_req, ) @@ -142,9 +143,9 @@ def resolve(self, max_rounds=10): @staticmethod def check_constraints(constraints): for constraint in constraints: - if constraint.link is not None and not constraint.editable: + if constraint.link is not None and constraint.link.url.startswith('file:') and not constraint.editable: msg = ( - "pip-compile does not support URLs as packages, unless " + "pip-compile does not support file URLs as packages, unless " "they are editable. Perhaps add -e option?" ) raise UnsupportedConstraint(msg, constraint) @@ -280,7 +281,7 @@ def get_best_match(self, ireq): Flask==0.10.1 => Flask==0.10.1 """ - if ireq.editable: + if ireq.editable or is_url_requirement(ireq, self.repository): # NOTE: it's much quicker to immediately return instead of # hitting the index server best_match = ireq @@ -303,14 +304,14 @@ def get_best_match(self, ireq): def _iter_dependencies(self, ireq): """ - Given a pinned or editable InstallRequirement, collects all the + Given a pinned, url, or editable InstallRequirement, collects all the secondary dependencies for them, either by looking them up in a local cache, or by reaching out to the repository. Editable requirements will never be looked up, as they may have changed at any time. """ - if ireq.editable: + if ireq.editable or is_url_requirement(ireq, self.repository): for dependency in self.repository.get_dependencies(ireq): yield dependency return @@ -345,5 +346,5 @@ def _iter_dependencies(self, ireq): yield install_req_from_line(dependency_string, constraint=ireq.constraint) def reverse_dependencies(self, ireqs): - non_editable = [ireq for ireq in ireqs if not ireq.editable] + non_editable = [ireq for ireq in ireqs if not (ireq.editable or is_url_requirement(ireq, self.repository))] return self.dependency_cache.reverse_dependencies(non_editable) diff --git a/piptools/sync.py b/piptools/sync.py index 7407ce8c0..eae6dc9a6 100644 --- a/piptools/sync.py +++ b/piptools/sync.py @@ -77,9 +77,9 @@ def merge(requirements, ignore_conflicts): by_key = {} for ireq in requirements: - if ireq.link is not None and not ireq.editable: + if ireq.link is not None and ireq.link.url.startswith('file:') and not ireq.editable: msg = ( - "pip-compile does not support URLs as packages, unless they are " + "pip-compile does not support file URLs as packages, unless they are " "editable. Perhaps add -e option?" ) raise UnsupportedConstraint(msg, ireq) diff --git a/piptools/utils.py b/piptools/utils.py index ad01fe0f8..99e42998b 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -59,6 +59,18 @@ def make_install_requirement(name, version, extras, constraint=False): ) +def is_url_requirement(ireq, repository=None): + """ + Finds if a requirement is a URL + """ + if not ireq.link or 'pypi.python.org' in ireq.link.url or ireq.link.url.startswith('file'): + return False + if repository is not None and hasattr(repository, 'finder'): + if any(index_url in ireq.link.url for index_url in repository.finder.index_urls): + return False + return True + + def format_requirement(ireq, marker=None, hashes=None): """ Generic formatter for pretty printing InstallRequirements to the terminal @@ -66,6 +78,8 @@ def format_requirement(ireq, marker=None, hashes=None): """ if ireq.editable: line = "-e {}".format(ireq.link.url) + elif is_url_requirement(ireq): + line = ireq.link.url else: line = str(ireq.req).lower() @@ -110,7 +124,10 @@ def is_pinned_requirement(ireq): if ireq.editable: return False - if len(ireq.specifier._specs) != 1: + try: + if len(ireq.specifier._specs) != 1: + return False + except Exception: return False op, version = next(iter(ireq.specifier._specs))._spec diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index cc2cd0291..e108bdfe7 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -273,6 +273,36 @@ def test_locally_available_editable_package_is_not_archived_in_cache_dir( assert not os.listdir(os.path.join(str(cache_dir), "pkgs")) +def test_url_package(tmpdir): + url_package = 'https://github.com/jazzband/pip-tools/archive/master.zip#egg=pip-tools' + runner = CliRunner() + with runner.isolated_filesystem(): + with open('requirements.in', 'w') as req_in: + req_in.write(url_package) + out = runner.invoke(cli, ['-n', '-r']) + assert out.exit_code == 0 + assert url_package in out.output + assert 'click' in out.output # dependency of pip-tools + + +def test_url_package_vcs(tmpdir): + vcs_package = ( + 'git+git://github.com/pytest-dev/pytest-django' + '@21492afc88a19d4ca01cd0ac392a5325b14f95c7' + '#egg=pytest-django' + ) + runner = CliRunner() + with runner.isolated_filesystem(): + with open('requirements.in', 'w') as req_in: + req_in.write(vcs_package) + out = runner.invoke(cli, ['-n', + '--rebuild']) + print(out.output) + assert out.exit_code == 0 + assert vcs_package in out.output + assert 'pytest' in out.output # dependency of pytest-django + + def test_input_file_without_extension(runner): """ piptools can compile a file without an extension, From de874f4eb159d5a64ac05e2d0e3bbf2f7e1f8d4d Mon Sep 17 00:00:00 2001 From: Jack Cushman Date: Fri, 3 May 2019 12:54:00 -0400 Subject: [PATCH 2/2] support URLs as packages --- piptools/_compat/__init__.py | 2 + piptools/exceptions.py | 10 --- piptools/repositories/base.py | 2 +- piptools/repositories/pypi.py | 24 +++++-- piptools/resolver.py | 21 ++---- piptools/scripts/compile.py | 15 +++-- piptools/sync.py | 41 ++++++++---- piptools/utils.py | 17 ++--- piptools/writer.py | 8 +-- tests/test_cli_compile.py | 116 ++++++++++++++++++++++++++-------- tests/test_resolver.py | 13 ---- tests/test_sync.py | 38 ++++++++--- tests/test_utils.py | 26 ++++++++ 13 files changed, 217 insertions(+), 116 deletions(-) diff --git a/piptools/_compat/__init__.py b/piptools/_compat/__init__.py index 75f70168f..9d23ddfc1 100644 --- a/piptools/_compat/__init__.py +++ b/piptools/_compat/__init__.py @@ -10,6 +10,7 @@ Command, FormatControl, InstallRequirement, + Link, PackageFinder, PyPI, RequirementSet, @@ -20,6 +21,7 @@ install_req_from_line, is_file_url, parse_requirements, + path_to_url, stdlib_pkgs, url_to_path, user_cache_dir, diff --git a/piptools/exceptions.py b/piptools/exceptions.py index d7572967c..76a409ef8 100644 --- a/piptools/exceptions.py +++ b/piptools/exceptions.py @@ -48,16 +48,6 @@ def __str__(self): return "\n".join(lines) -class UnsupportedConstraint(PipToolsError): - def __init__(self, message, constraint): - super(UnsupportedConstraint, self).__init__(message) - self.constraint = constraint - - def __str__(self): - message = super(UnsupportedConstraint, self).__str__() - return "{} (constraint was: {})".format(message, str(self.constraint)) - - class IncompatibleRequirements(PipToolsError): def __init__(self, ireq_a, ireq_b): self.ireq_a = ireq_a diff --git a/piptools/repositories/base.py b/piptools/repositories/base.py index 1339c179c..dd73ff32e 100644 --- a/piptools/repositories/base.py +++ b/piptools/repositories/base.py @@ -25,7 +25,7 @@ def find_best_match(self, ireq): @abstractmethod def get_dependencies(self, ireq): """ - Given a pinned or an editable InstallRequirement, returns a set of + Given a pinned, URL, or editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). They indicate the secondary dependencies for the given requirement. """ diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index c5070e168..1397b97cf 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -12,6 +12,7 @@ from .._compat import ( FAVORITE_HASH, + Link, PackageFinder, PyPI, RequirementSet, @@ -19,6 +20,7 @@ Wheel, contextlib, is_file_url, + path_to_url, url_to_path, ) from ..cache import CACHE_DIR @@ -137,7 +139,7 @@ def find_best_match(self, ireq, prereleases=None): Returns a Version object that indicates the best match for the given InstallRequirement according to the external repository. """ - if ireq.editable or is_url_requirement(ireq, self): + if ireq.editable or is_url_requirement(ireq): return ireq # return itself as the best match all_candidates = self.find_all_candidates(ireq.name) @@ -229,13 +231,17 @@ def resolve_reqs(self, download_dir, ireq, wheel_cache): def get_dependencies(self, ireq): """ - Given a pinned, a url, or an editable InstallRequirement, returns a set of + Given a pinned, URL, or editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). They indicate the secondary dependencies for the given requirement. """ - if not (ireq.editable or is_url_requirement(ireq, self) or is_pinned_requirement(ireq)): + if not ( + ireq.editable or is_url_requirement(ireq) or is_pinned_requirement(ireq) + ): raise TypeError( - "Expected url, pinned or editable InstallRequirement, got {}".format(ireq) + "Expected url, pinned or editable InstallRequirement, got {}".format( + ireq + ) ) if ireq not in self._dependencies_cache: @@ -282,6 +288,16 @@ def get_hashes(self, ireq): if ireq.editable: return set() + if is_url_requirement(ireq): + # url requirements may have been previously downloaded and cached + # locally by self.resolve_reqs() + cached_path = os.path.join(self._download_dir, ireq.link.filename) + if os.path.exists(cached_path): + cached_link = Link(path_to_url(cached_path)) + else: + cached_link = ireq.link + return {self._get_file_hash(cached_link)} + if not is_pinned_requirement(ireq): raise TypeError("Expected pinned requirement, got {}".format(ireq)) diff --git a/piptools/resolver.py b/piptools/resolver.py index c4fdf4f60..a703a2840 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -9,7 +9,6 @@ from . import click from ._compat import install_req_from_line from .cache import DependencyCache -from .exceptions import UnsupportedConstraint from .logging import log from .utils import ( UNSAFE_PACKAGES, @@ -102,8 +101,6 @@ def resolve(self, max_rounds=10): self.dependency_cache.clear() self.repository.clear_caches() - self.check_constraints(chain(self.our_constraints, self.their_constraints)) - # Ignore existing packages os.environ[str("PIP_EXISTS_ACTION")] = str( "i" @@ -140,16 +137,6 @@ def resolve(self, max_rounds=10): # Only include hard requirements and not pip constraints return {req for req in best_matches if not req.constraint} - @staticmethod - def check_constraints(constraints): - for constraint in constraints: - if constraint.link is not None and constraint.link.url.startswith('file:') and not constraint.editable: - msg = ( - "pip-compile does not support file URLs as packages, unless " - "they are editable. Perhaps add -e option?" - ) - raise UnsupportedConstraint(msg, constraint) - def _group_constraints(self, constraints): """ Groups constraints (remember, InstallRequirements!) by their key name, @@ -281,7 +268,7 @@ def get_best_match(self, ireq): Flask==0.10.1 => Flask==0.10.1 """ - if ireq.editable or is_url_requirement(ireq, self.repository): + if ireq.editable or is_url_requirement(ireq): # NOTE: it's much quicker to immediately return instead of # hitting the index server best_match = ireq @@ -311,7 +298,7 @@ def _iter_dependencies(self, ireq): Editable requirements will never be looked up, as they may have changed at any time. """ - if ireq.editable or is_url_requirement(ireq, self.repository): + if ireq.editable or is_url_requirement(ireq): for dependency in self.repository.get_dependencies(ireq): yield dependency return @@ -346,5 +333,7 @@ def _iter_dependencies(self, ireq): yield install_req_from_line(dependency_string, constraint=ireq.constraint) def reverse_dependencies(self, ireqs): - non_editable = [ireq for ireq in ireqs if not (ireq.editable or is_url_requirement(ireq, self.repository))] + non_editable = [ + ireq for ireq in ireqs if not (ireq.editable or is_url_requirement(ireq)) + ] return self.dependency_cache.reverse_dependencies(non_editable) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 896de7a75..8e4e6a0ae 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -14,7 +14,13 @@ from ..pip import get_pip_command, pip_defaults from ..repositories import LocalRequirementsRepository, PyPIRepository from ..resolver import Resolver -from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_req +from ..utils import ( + UNSAFE_PACKAGES, + dedup, + is_pinned_requirement, + key_from_ireq, + key_from_req, +) from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = "requirements.in" @@ -338,9 +344,6 @@ def cli( for find_link in repository.finder.find_links: log.debug(" -f {}".format(find_link)) - # Check the given base set of constraints first - Resolver.check_constraints(constraints) - try: resolver = Resolver( constraints, @@ -413,10 +416,10 @@ def cli( unsafe_requirements=resolver.unsafe_constraints, reverse_dependencies=reverse_dependencies, primary_packages={ - key_from_req(ireq.req) for ireq in constraints if not ireq.constraint + key_from_ireq(ireq) for ireq in constraints if not ireq.constraint }, markers={ - key_from_req(ireq.req): ireq.markers for ireq in constraints if ireq.markers + key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers }, hashes=hashes, ) diff --git a/piptools/sync.py b/piptools/sync.py index eae6dc9a6..a53b1dc44 100644 --- a/piptools/sync.py +++ b/piptools/sync.py @@ -5,17 +5,17 @@ from subprocess import check_call # nosec from . import click -from .exceptions import IncompatibleRequirements, UnsupportedConstraint +from ._compat import DEV_PKGS, stdlib_pkgs +from .exceptions import IncompatibleRequirements from .utils import ( flat_map, format_requirement, get_hashes_from_ireq, + is_url_requirement, key_from_ireq, key_from_req, ) -from piptools._compat import DEV_PKGS, stdlib_pkgs - PACKAGES_TO_IGNORE = ( ["-markerlib", "pip", "pip-tools", "pip-review", "pkg-resources"] + list(stdlib_pkgs) @@ -77,14 +77,10 @@ def merge(requirements, ignore_conflicts): by_key = {} for ireq in requirements: - if ireq.link is not None and ireq.link.url.startswith('file:') and not ireq.editable: - msg = ( - "pip-compile does not support file URLs as packages, unless they are " - "editable. Perhaps add -e option?" - ) - raise UnsupportedConstraint(msg, ireq) - - key = ireq.link or key_from_req(ireq.req) + # Limitation: URL requirements are merged by precise string match, so + # "file:///example.zip#egg=example", "file:///example.zip", and + # "example==1.0" will not merge with each other + key = key_from_ireq(ireq) if not ignore_conflicts: existing_ireq = by_key.get(key) @@ -96,16 +92,35 @@ def merge(requirements, ignore_conflicts): # TODO: Always pick the largest specifier in case of a conflict by_key[key] = ireq - return by_key.values() +def diff_key_from_ireq(ireq): + """ + Calculate a key for comparing a compiled requirement with installed modules. + For URL requirements, only provide a useful key if the url includes + #egg=name==version, which will set ireq.req.name and ireq.specifier. + Otherwise return ireq.link so the key will not match and the package will + reinstall. Reinstall is necessary to ensure that packages will reinstall + if the URL is changed but the version is not. + """ + if is_url_requirement(ireq): + if ( + ireq.req + and (getattr(ireq.req, "key", None) or getattr(ireq.req, "name", None)) + and ireq.specifier + ): + return key_from_ireq(ireq) + return str(ireq.link) + return key_from_ireq(ireq) + + def diff(compiled_requirements, installed_dists): """ Calculate which packages should be installed or uninstalled, given a set of compiled requirements and a list of currently installed modules. """ - requirements_lut = {r.link or key_from_req(r.req): r for r in compiled_requirements} + requirements_lut = {diff_key_from_ireq(r): r for r in compiled_requirements} satisfied = set() # holds keys to_install = set() # holds InstallRequirement objects diff --git a/piptools/utils.py b/piptools/utils.py index 99e42998b..c0b1bdbaf 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -59,16 +59,12 @@ def make_install_requirement(name, version, extras, constraint=False): ) -def is_url_requirement(ireq, repository=None): +def is_url_requirement(ireq): """ - Finds if a requirement is a URL + Return True if requirement was specified as a path or URL. + ireq.original_link will have been set by InstallRequirement.__init__ """ - if not ireq.link or 'pypi.python.org' in ireq.link.url or ireq.link.url.startswith('file'): - return False - if repository is not None and hasattr(repository, 'finder'): - if any(index_url in ireq.link.url for index_url in repository.finder.index_urls): - return False - return True + return bool(ireq.original_link) def format_requirement(ireq, marker=None, hashes=None): @@ -124,10 +120,7 @@ def is_pinned_requirement(ireq): if ireq.editable: return False - try: - if len(ireq.specifier._specs) != 1: - return False - except Exception: + if ireq.req is None or len(ireq.specifier._specs) != 1: return False op, version = next(iter(ireq.specifier._specs))._spec diff --git a/piptools/writer.py b/piptools/writer.py index 18a715b82..92355519f 100644 --- a/piptools/writer.py +++ b/piptools/writer.py @@ -11,7 +11,7 @@ dedup, format_requirement, get_compile_command, - key_from_req, + key_from_ireq, ) @@ -129,7 +129,7 @@ def _iter_lines( ireq, reverse_dependencies, primary_packages, - markers.get(key_from_req(ireq.req)), + markers.get(key_from_ireq(ireq)), hashes=hashes, ) yield line @@ -147,7 +147,7 @@ def _iter_lines( ireq, reverse_dependencies, primary_packages, - marker=markers.get(key_from_req(ireq.req)), + marker=markers.get(key_from_ireq(ireq)), hashes=hashes, ) if not self.allow_unsafe: @@ -185,7 +185,7 @@ def _format_requirement( line = format_requirement(ireq, marker=marker, hashes=ireq_hashes) - if not self.annotate or key_from_req(ireq.req) in primary_packages: + if not self.annotate or key_from_ireq(ireq) in primary_packages: return line # Annotate what packages this package is required by diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index e108bdfe7..55af6c61e 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -8,6 +8,7 @@ from click.testing import CliRunner from pip import __version__ as pip_version from pip._vendor.packaging.version import parse as parse_version +from pytest import mark from .utils import invoke @@ -273,34 +274,76 @@ def test_locally_available_editable_package_is_not_archived_in_cache_dir( assert not os.listdir(os.path.join(str(cache_dir), "pkgs")) -def test_url_package(tmpdir): - url_package = 'https://github.com/jazzband/pip-tools/archive/master.zip#egg=pip-tools' - runner = CliRunner() - with runner.isolated_filesystem(): - with open('requirements.in', 'w') as req_in: - req_in.write(url_package) - out = runner.invoke(cli, ['-n', '-r']) - assert out.exit_code == 0 - assert url_package in out.output - assert 'click' in out.output # dependency of pip-tools - - -def test_url_package_vcs(tmpdir): - vcs_package = ( - 'git+git://github.com/pytest-dev/pytest-django' - '@21492afc88a19d4ca01cd0ac392a5325b14f95c7' - '#egg=pytest-django' - ) - runner = CliRunner() - with runner.isolated_filesystem(): - with open('requirements.in', 'w') as req_in: - req_in.write(vcs_package) - out = runner.invoke(cli, ['-n', - '--rebuild']) - print(out.output) - assert out.exit_code == 0 - assert vcs_package in out.output - assert 'pytest' in out.output # dependency of pytest-django +@mark.parametrize( + ("line", "dependency", "rewritten_line"), + [ + # zip URL + # use pip-tools version prior to its use of setuptools_scm, + # which is incompatible with https: install + ( + "https://github.com/jazzband/pip-tools/archive/" + "7d86c8d3ecd1faa6be11c7ddc6b29a30ffd1dae3.zip", + "\nclick==", + None, + ), + # scm URL + ( + "git+git://github.com/jazzband/pip-tools@" + "7d86c8d3ecd1faa6be11c7ddc6b29a30ffd1dae3", + "\nclick==", + None, + ), + # wheel URL + ( + "https://files.pythonhosted.org/packages/06/96/" + "89872db07ae70770fba97205b0737c17ef013d0d1c790" + "899c16bb8bac419/pip_tools-3.6.1-py2.py3-none-any.whl", + "\nclick==", + None, + ), + # file:// wheel URL + ( + path_to_url( + os.path.join( + TEST_DATA_PATH, + "minimal_wheels/small_fake_with_deps-0.1-py2.py3-none-any.whl", + ) + ), + "\nsix==", + None, + ), + # file:// directory + ( + path_to_url(os.path.join(TEST_DATA_PATH, "small_fake_package")), + "\nsix==", + None, + ), + # bare path + # will be rewritten to file:// URL + ( + os.path.join( + TEST_DATA_PATH, + "minimal_wheels/small_fake_with_deps-0.1-py2.py3-none-any.whl", + ), + "\nsix==", + path_to_url( + os.path.join( + TEST_DATA_PATH, + "minimal_wheels/small_fake_with_deps-0.1-py2.py3-none-any.whl", + ) + ), + ), + ], +) +def test_url_package(runner, line, dependency, rewritten_line): + if rewritten_line is None: + rewritten_line = line + with open("requirements.in", "w") as req_in: + req_in.write(line) + out = runner.invoke(cli, ["-n", "--rebuild"]) + assert out.exit_code == 0 + assert rewritten_line in out.output + assert dependency in out.output def test_input_file_without_extension(runner): @@ -381,6 +424,23 @@ def test_generate_hashes_with_editable(runner): assert expected in out.output +def test_generate_hashes_with_url(runner): + with open("requirements.in", "w") as fp: + fp.write( + "https://github.com/jazzband/pip-tools/archive/" + "7d86c8d3ecd1faa6be11c7ddc6b29a30ffd1dae3.zip#egg=pip-tools\n" + ) + out = runner.invoke(cli, ["--generate-hashes"]) + expected = ( + "https://github.com/jazzband/pip-tools/archive/" + "7d86c8d3ecd1faa6be11c7ddc6b29a30ffd1dae3.zip#egg=pip-tools \\\n" + " --hash=sha256:d24de92e18ad5bf291f25cfcdcf" + "0171be6fa70d01d0bef9eeda356b8549715e7\n" + ) + assert out.exit_code == 0 + assert expected in out.output + + def test_generate_hashes_verbose(runner): """ The hashes generation process should show a progress. diff --git a/tests/test_resolver.py b/tests/test_resolver.py index e4ef0e21b..46eda846a 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -1,7 +1,5 @@ import pytest -from piptools.exceptions import UnsupportedConstraint - @pytest.mark.parametrize( ("input", "expected", "prereleases"), @@ -171,17 +169,6 @@ def test_resolver__max_number_rounds_reached(resolver, from_line): resolver(input).resolve(max_rounds=0) -def test_resolver__check_constraints(resolver, from_line): - """ - Resolver should not support non-editable URLs as packages. - """ - input = [from_line("django"), from_line("https://example.com/#egg=example")] - with pytest.raises( - UnsupportedConstraint, match="pip-compile does not support URLs as packages" - ): - resolver(input).resolve() - - def test_iter_dependencies(resolver, from_line): """ Dependencies should be pinned or editable. diff --git a/tests/test_sync.py b/tests/test_sync.py index 592c38428..e690a7b12 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -7,7 +7,7 @@ import mock import pytest -from piptools.exceptions import IncompatibleRequirements, UnsupportedConstraint +from piptools.exceptions import IncompatibleRequirements from piptools.sync import dependency_tree, diff, merge, sync @@ -98,17 +98,16 @@ def test_merge(from_line): ) -def test_merge_non_editable_url(from_line): - """ - Non-editable URLs are not supported. - """ +def test_merge_urls(from_line): requirements = [ - from_line("django==1.8"), - from_line("https://example.com/#egg=example"), + from_line("file:///example.zip#egg=example==1.0"), + from_line("example==1.0"), + from_line("file:///unrelated.zip"), ] - with pytest.raises(UnsupportedConstraint): - merge(requirements, ignore_conflicts=True) + assert Counter(requirements[1:]) == Counter( + merge(requirements, ignore_conflicts=False) + ) def test_diff_should_do_nothing(): @@ -251,6 +250,27 @@ def test_diff_with_editable(fake_dist, from_editable): assert str(package.link) == _get_file_url(path_to_package) +def test_diff_with_matching_url_versions(fake_dist, from_line): + # if URL version is explicitly provided, use it to avoid reinstalling + installed = [fake_dist("example==1.0")] + reqs = [from_line("file:///example.zip#egg=example==1.0")] + + to_install, to_uninstall = diff(reqs, installed) + assert to_install == set() + assert to_uninstall == set() + + +def test_diff_with_no_url_versions(fake_dist, from_line): + # if URL version is not provided, assume the contents have + # changed and reinstall + installed = [fake_dist("example==1.0")] + reqs = [from_line("file:///example.zip#egg=example")] + + to_install, to_uninstall = diff(reqs, installed) + assert to_install == set(reqs) + assert to_uninstall == {"example"} + + def test_sync_install_temporary_requirement_file( from_line, from_editable, mocked_tmp_req_file ): diff --git a/tests/test_utils.py b/tests/test_utils.py index 3bd1281fb..e8af29213 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,6 +19,7 @@ get_compile_command, get_hashes_from_ireq, is_pinned_requirement, + is_url_requirement, name_from_req, ) @@ -28,6 +29,11 @@ def test_format_requirement(from_line): assert format_requirement(ireq) == "test==1.2" +def test_format_requirement_url(from_line): + ireq = from_line("https://example.com/example.zip") + assert format_requirement(ireq) == "https://example.com/example.zip" + + def test_format_requirement_editable_vcs(from_editable): ireq = from_editable("git+git://fake.org/x/y.git#egg=y") assert format_requirement(ireq) == "-e git+git://fake.org/x/y.git#egg=y" @@ -149,6 +155,8 @@ def test_get_hashes_from_ireq(from_line): ("django>1.8", False), ("django~=1.8", False), ("django==1.*", False), + ("file:///example.zip", False), + ("https://example.com/example.zip", False), ], ) def test_is_pinned_requirement(from_line, line, expected): @@ -161,6 +169,24 @@ def test_is_pinned_requirement_editable(from_editable): assert not is_pinned_requirement(ireq) +@mark.parametrize( + ("line", "expected"), + [ + ("django==1.8", False), + ("django", False), + ("file:///example.zip", True), + ("https://example.com/example.zip", True), + ("https://example.com/example.zip#egg=example", True), + ("git+git://github.com/jazzband/pip-tools@master", True), + ("../example.zip", True), + ("/example.zip", True), + ], +) +def test_is_url_requirement(from_line, line, expected): + ireq = from_line(line) + assert is_url_requirement(ireq) is expected + + def test_name_from_req(from_line): ireq = from_line("django==1.8") assert name_from_req(ireq.req) == "django"