diff --git a/piptools/_compat/__init__.py b/piptools/_compat/__init__.py index 75f70168f..9d23ddfc1 100644 --- a/piptools/_compat/__init__.py +++ b/piptools/_compat/__init__.py @@ -10,6 +10,7 @@ Command, FormatControl, InstallRequirement, + Link, PackageFinder, PyPI, RequirementSet, @@ -20,6 +21,7 @@ install_req_from_line, is_file_url, parse_requirements, + path_to_url, stdlib_pkgs, url_to_path, user_cache_dir, diff --git a/piptools/exceptions.py b/piptools/exceptions.py index d7572967c..76a409ef8 100644 --- a/piptools/exceptions.py +++ b/piptools/exceptions.py @@ -48,16 +48,6 @@ def __str__(self): return "\n".join(lines) -class UnsupportedConstraint(PipToolsError): - def __init__(self, message, constraint): - super(UnsupportedConstraint, self).__init__(message) - self.constraint = constraint - - def __str__(self): - message = super(UnsupportedConstraint, self).__str__() - return "{} (constraint was: {})".format(message, str(self.constraint)) - - class IncompatibleRequirements(PipToolsError): def __init__(self, ireq_a, ireq_b): self.ireq_a = ireq_a diff --git a/piptools/repositories/base.py b/piptools/repositories/base.py index 1339c179c..dd73ff32e 100644 --- a/piptools/repositories/base.py +++ b/piptools/repositories/base.py @@ -25,7 +25,7 @@ def find_best_match(self, ireq): @abstractmethod def get_dependencies(self, ireq): """ - Given a pinned or an editable InstallRequirement, returns a set of + Given a pinned, URL, or editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). They indicate the secondary dependencies for the given requirement. """ diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index 3a332c1c5..1397b97cf 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -12,6 +12,7 @@ from .._compat import ( FAVORITE_HASH, + Link, PackageFinder, PyPI, RequirementSet, @@ -19,6 +20,7 @@ Wheel, contextlib, is_file_url, + path_to_url, url_to_path, ) from ..cache import CACHE_DIR @@ -28,6 +30,7 @@ from ..utils import ( fs_str, is_pinned_requirement, + is_url_requirement, lookup_table, make_install_requirement, ) @@ -136,7 +139,7 @@ def find_best_match(self, ireq, prereleases=None): Returns a Version object that indicates the best match for the given InstallRequirement according to the external repository. """ - if ireq.editable: + if ireq.editable or is_url_requirement(ireq): return ireq # return itself as the best match all_candidates = self.find_all_candidates(ireq.name) @@ -228,13 +231,17 @@ def resolve_reqs(self, download_dir, ireq, wheel_cache): def get_dependencies(self, ireq): """ - Given a pinned or an editable InstallRequirement, returns a set of + Given a pinned, URL, or editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). They indicate the secondary dependencies for the given requirement. """ - if not (ireq.editable or is_pinned_requirement(ireq)): + if not ( + ireq.editable or is_url_requirement(ireq) or is_pinned_requirement(ireq) + ): raise TypeError( - "Expected pinned or editable InstallRequirement, got {}".format(ireq) + "Expected url, pinned or editable InstallRequirement, got {}".format( + ireq + ) ) if ireq not in self._dependencies_cache: @@ -281,6 +288,16 @@ def get_hashes(self, ireq): if ireq.editable: return set() + if is_url_requirement(ireq): + # url requirements may have been previously downloaded and cached + # locally by self.resolve_reqs() + cached_path = os.path.join(self._download_dir, ireq.link.filename) + if os.path.exists(cached_path): + cached_link = Link(path_to_url(cached_path)) + else: + cached_link = ireq.link + return {self._get_file_hash(cached_link)} + if not is_pinned_requirement(ireq): raise TypeError("Expected pinned requirement, got {}".format(ireq)) diff --git a/piptools/resolver.py b/piptools/resolver.py index 764cb6b2f..a703a2840 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -9,7 +9,6 @@ from . import click from ._compat import install_req_from_line from .cache import DependencyCache -from .exceptions import UnsupportedConstraint from .logging import log from .utils import ( UNSAFE_PACKAGES, @@ -17,6 +16,7 @@ format_specifier, full_groupby, is_pinned_requirement, + is_url_requirement, key_from_ireq, key_from_req, ) @@ -101,8 +101,6 @@ def resolve(self, max_rounds=10): self.dependency_cache.clear() self.repository.clear_caches() - self.check_constraints(chain(self.our_constraints, self.their_constraints)) - # Ignore existing packages os.environ[str("PIP_EXISTS_ACTION")] = str( "i" @@ -139,16 +137,6 @@ def resolve(self, max_rounds=10): # Only include hard requirements and not pip constraints return {req for req in best_matches if not req.constraint} - @staticmethod - def check_constraints(constraints): - for constraint in constraints: - if constraint.link is not None and not constraint.editable: - msg = ( - "pip-compile does not support URLs as packages, unless " - "they are editable. Perhaps add -e option?" - ) - raise UnsupportedConstraint(msg, constraint) - def _group_constraints(self, constraints): """ Groups constraints (remember, InstallRequirements!) by their key name, @@ -280,7 +268,7 @@ def get_best_match(self, ireq): Flask==0.10.1 => Flask==0.10.1 """ - if ireq.editable: + if ireq.editable or is_url_requirement(ireq): # NOTE: it's much quicker to immediately return instead of # hitting the index server best_match = ireq @@ -303,14 +291,14 @@ def get_best_match(self, ireq): def _iter_dependencies(self, ireq): """ - Given a pinned or editable InstallRequirement, collects all the + Given a pinned, url, or editable InstallRequirement, collects all the secondary dependencies for them, either by looking them up in a local cache, or by reaching out to the repository. Editable requirements will never be looked up, as they may have changed at any time. """ - if ireq.editable: + if ireq.editable or is_url_requirement(ireq): for dependency in self.repository.get_dependencies(ireq): yield dependency return @@ -345,5 +333,7 @@ def _iter_dependencies(self, ireq): yield install_req_from_line(dependency_string, constraint=ireq.constraint) def reverse_dependencies(self, ireqs): - non_editable = [ireq for ireq in ireqs if not ireq.editable] + non_editable = [ + ireq for ireq in ireqs if not (ireq.editable or is_url_requirement(ireq)) + ] return self.dependency_cache.reverse_dependencies(non_editable) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 896de7a75..8e4e6a0ae 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -14,7 +14,13 @@ from ..pip import get_pip_command, pip_defaults from ..repositories import LocalRequirementsRepository, PyPIRepository from ..resolver import Resolver -from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_req +from ..utils import ( + UNSAFE_PACKAGES, + dedup, + is_pinned_requirement, + key_from_ireq, + key_from_req, +) from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = "requirements.in" @@ -338,9 +344,6 @@ def cli( for find_link in repository.finder.find_links: log.debug(" -f {}".format(find_link)) - # Check the given base set of constraints first - Resolver.check_constraints(constraints) - try: resolver = Resolver( constraints, @@ -413,10 +416,10 @@ def cli( unsafe_requirements=resolver.unsafe_constraints, reverse_dependencies=reverse_dependencies, primary_packages={ - key_from_req(ireq.req) for ireq in constraints if not ireq.constraint + key_from_ireq(ireq) for ireq in constraints if not ireq.constraint }, markers={ - key_from_req(ireq.req): ireq.markers for ireq in constraints if ireq.markers + key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers }, hashes=hashes, ) diff --git a/piptools/sync.py b/piptools/sync.py index 7407ce8c0..a53b1dc44 100644 --- a/piptools/sync.py +++ b/piptools/sync.py @@ -5,17 +5,17 @@ from subprocess import check_call # nosec from . import click -from .exceptions import IncompatibleRequirements, UnsupportedConstraint +from ._compat import DEV_PKGS, stdlib_pkgs +from .exceptions import IncompatibleRequirements from .utils import ( flat_map, format_requirement, get_hashes_from_ireq, + is_url_requirement, key_from_ireq, key_from_req, ) -from piptools._compat import DEV_PKGS, stdlib_pkgs - PACKAGES_TO_IGNORE = ( ["-markerlib", "pip", "pip-tools", "pip-review", "pkg-resources"] + list(stdlib_pkgs) @@ -77,14 +77,10 @@ def merge(requirements, ignore_conflicts): by_key = {} for ireq in requirements: - if ireq.link is not None and not ireq.editable: - msg = ( - "pip-compile does not support URLs as packages, unless they are " - "editable. Perhaps add -e option?" - ) - raise UnsupportedConstraint(msg, ireq) - - key = ireq.link or key_from_req(ireq.req) + # Limitation: URL requirements are merged by precise string match, so + # "file:///example.zip#egg=example", "file:///example.zip", and + # "example==1.0" will not merge with each other + key = key_from_ireq(ireq) if not ignore_conflicts: existing_ireq = by_key.get(key) @@ -96,16 +92,35 @@ def merge(requirements, ignore_conflicts): # TODO: Always pick the largest specifier in case of a conflict by_key[key] = ireq - return by_key.values() +def diff_key_from_ireq(ireq): + """ + Calculate a key for comparing a compiled requirement with installed modules. + For URL requirements, only provide a useful key if the url includes + #egg=name==version, which will set ireq.req.name and ireq.specifier. + Otherwise return ireq.link so the key will not match and the package will + reinstall. Reinstall is necessary to ensure that packages will reinstall + if the URL is changed but the version is not. + """ + if is_url_requirement(ireq): + if ( + ireq.req + and (getattr(ireq.req, "key", None) or getattr(ireq.req, "name", None)) + and ireq.specifier + ): + return key_from_ireq(ireq) + return str(ireq.link) + return key_from_ireq(ireq) + + def diff(compiled_requirements, installed_dists): """ Calculate which packages should be installed or uninstalled, given a set of compiled requirements and a list of currently installed modules. """ - requirements_lut = {r.link or key_from_req(r.req): r for r in compiled_requirements} + requirements_lut = {diff_key_from_ireq(r): r for r in compiled_requirements} satisfied = set() # holds keys to_install = set() # holds InstallRequirement objects diff --git a/piptools/utils.py b/piptools/utils.py index ad01fe0f8..c0b1bdbaf 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -59,6 +59,14 @@ def make_install_requirement(name, version, extras, constraint=False): ) +def is_url_requirement(ireq): + """ + Return True if requirement was specified as a path or URL. + ireq.original_link will have been set by InstallRequirement.__init__ + """ + return bool(ireq.original_link) + + def format_requirement(ireq, marker=None, hashes=None): """ Generic formatter for pretty printing InstallRequirements to the terminal @@ -66,6 +74,8 @@ def format_requirement(ireq, marker=None, hashes=None): """ if ireq.editable: line = "-e {}".format(ireq.link.url) + elif is_url_requirement(ireq): + line = ireq.link.url else: line = str(ireq.req).lower() @@ -110,7 +120,7 @@ def is_pinned_requirement(ireq): if ireq.editable: return False - if len(ireq.specifier._specs) != 1: + if ireq.req is None or len(ireq.specifier._specs) != 1: return False op, version = next(iter(ireq.specifier._specs))._spec diff --git a/piptools/writer.py b/piptools/writer.py index 18a715b82..92355519f 100644 --- a/piptools/writer.py +++ b/piptools/writer.py @@ -11,7 +11,7 @@ dedup, format_requirement, get_compile_command, - key_from_req, + key_from_ireq, ) @@ -129,7 +129,7 @@ def _iter_lines( ireq, reverse_dependencies, primary_packages, - markers.get(key_from_req(ireq.req)), + markers.get(key_from_ireq(ireq)), hashes=hashes, ) yield line @@ -147,7 +147,7 @@ def _iter_lines( ireq, reverse_dependencies, primary_packages, - marker=markers.get(key_from_req(ireq.req)), + marker=markers.get(key_from_ireq(ireq)), hashes=hashes, ) if not self.allow_unsafe: @@ -185,7 +185,7 @@ def _format_requirement( line = format_requirement(ireq, marker=marker, hashes=ireq_hashes) - if not self.annotate or key_from_req(ireq.req) in primary_packages: + if not self.annotate or key_from_ireq(ireq) in primary_packages: return line # Annotate what packages this package is required by diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index cc2cd0291..55af6c61e 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -8,6 +8,7 @@ from click.testing import CliRunner from pip import __version__ as pip_version from pip._vendor.packaging.version import parse as parse_version +from pytest import mark from .utils import invoke @@ -273,6 +274,78 @@ def test_locally_available_editable_package_is_not_archived_in_cache_dir( assert not os.listdir(os.path.join(str(cache_dir), "pkgs")) +@mark.parametrize( + ("line", "dependency", "rewritten_line"), + [ + # zip URL + # use pip-tools version prior to its use of setuptools_scm, + # which is incompatible with https: install + ( + "https://github.com/jazzband/pip-tools/archive/" + "7d86c8d3ecd1faa6be11c7ddc6b29a30ffd1dae3.zip", + "\nclick==", + None, + ), + # scm URL + ( + "git+git://github.com/jazzband/pip-tools@" + "7d86c8d3ecd1faa6be11c7ddc6b29a30ffd1dae3", + "\nclick==", + None, + ), + # wheel URL + ( + "https://files.pythonhosted.org/packages/06/96/" + "89872db07ae70770fba97205b0737c17ef013d0d1c790" + "899c16bb8bac419/pip_tools-3.6.1-py2.py3-none-any.whl", + "\nclick==", + None, + ), + # file:// wheel URL + ( + path_to_url( + os.path.join( + TEST_DATA_PATH, + "minimal_wheels/small_fake_with_deps-0.1-py2.py3-none-any.whl", + ) + ), + "\nsix==", + None, + ), + # file:// directory + ( + path_to_url(os.path.join(TEST_DATA_PATH, "small_fake_package")), + "\nsix==", + None, + ), + # bare path + # will be rewritten to file:// URL + ( + os.path.join( + TEST_DATA_PATH, + "minimal_wheels/small_fake_with_deps-0.1-py2.py3-none-any.whl", + ), + "\nsix==", + path_to_url( + os.path.join( + TEST_DATA_PATH, + "minimal_wheels/small_fake_with_deps-0.1-py2.py3-none-any.whl", + ) + ), + ), + ], +) +def test_url_package(runner, line, dependency, rewritten_line): + if rewritten_line is None: + rewritten_line = line + with open("requirements.in", "w") as req_in: + req_in.write(line) + out = runner.invoke(cli, ["-n", "--rebuild"]) + assert out.exit_code == 0 + assert rewritten_line in out.output + assert dependency in out.output + + def test_input_file_without_extension(runner): """ piptools can compile a file without an extension, @@ -351,6 +424,23 @@ def test_generate_hashes_with_editable(runner): assert expected in out.output +def test_generate_hashes_with_url(runner): + with open("requirements.in", "w") as fp: + fp.write( + "https://github.com/jazzband/pip-tools/archive/" + "7d86c8d3ecd1faa6be11c7ddc6b29a30ffd1dae3.zip#egg=pip-tools\n" + ) + out = runner.invoke(cli, ["--generate-hashes"]) + expected = ( + "https://github.com/jazzband/pip-tools/archive/" + "7d86c8d3ecd1faa6be11c7ddc6b29a30ffd1dae3.zip#egg=pip-tools \\\n" + " --hash=sha256:d24de92e18ad5bf291f25cfcdcf" + "0171be6fa70d01d0bef9eeda356b8549715e7\n" + ) + assert out.exit_code == 0 + assert expected in out.output + + def test_generate_hashes_verbose(runner): """ The hashes generation process should show a progress. diff --git a/tests/test_resolver.py b/tests/test_resolver.py index e4ef0e21b..46eda846a 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -1,7 +1,5 @@ import pytest -from piptools.exceptions import UnsupportedConstraint - @pytest.mark.parametrize( ("input", "expected", "prereleases"), @@ -171,17 +169,6 @@ def test_resolver__max_number_rounds_reached(resolver, from_line): resolver(input).resolve(max_rounds=0) -def test_resolver__check_constraints(resolver, from_line): - """ - Resolver should not support non-editable URLs as packages. - """ - input = [from_line("django"), from_line("https://example.com/#egg=example")] - with pytest.raises( - UnsupportedConstraint, match="pip-compile does not support URLs as packages" - ): - resolver(input).resolve() - - def test_iter_dependencies(resolver, from_line): """ Dependencies should be pinned or editable. diff --git a/tests/test_sync.py b/tests/test_sync.py index 592c38428..e690a7b12 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -7,7 +7,7 @@ import mock import pytest -from piptools.exceptions import IncompatibleRequirements, UnsupportedConstraint +from piptools.exceptions import IncompatibleRequirements from piptools.sync import dependency_tree, diff, merge, sync @@ -98,17 +98,16 @@ def test_merge(from_line): ) -def test_merge_non_editable_url(from_line): - """ - Non-editable URLs are not supported. - """ +def test_merge_urls(from_line): requirements = [ - from_line("django==1.8"), - from_line("https://example.com/#egg=example"), + from_line("file:///example.zip#egg=example==1.0"), + from_line("example==1.0"), + from_line("file:///unrelated.zip"), ] - with pytest.raises(UnsupportedConstraint): - merge(requirements, ignore_conflicts=True) + assert Counter(requirements[1:]) == Counter( + merge(requirements, ignore_conflicts=False) + ) def test_diff_should_do_nothing(): @@ -251,6 +250,27 @@ def test_diff_with_editable(fake_dist, from_editable): assert str(package.link) == _get_file_url(path_to_package) +def test_diff_with_matching_url_versions(fake_dist, from_line): + # if URL version is explicitly provided, use it to avoid reinstalling + installed = [fake_dist("example==1.0")] + reqs = [from_line("file:///example.zip#egg=example==1.0")] + + to_install, to_uninstall = diff(reqs, installed) + assert to_install == set() + assert to_uninstall == set() + + +def test_diff_with_no_url_versions(fake_dist, from_line): + # if URL version is not provided, assume the contents have + # changed and reinstall + installed = [fake_dist("example==1.0")] + reqs = [from_line("file:///example.zip#egg=example")] + + to_install, to_uninstall = diff(reqs, installed) + assert to_install == set(reqs) + assert to_uninstall == {"example"} + + def test_sync_install_temporary_requirement_file( from_line, from_editable, mocked_tmp_req_file ): diff --git a/tests/test_utils.py b/tests/test_utils.py index 3bd1281fb..e8af29213 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,6 +19,7 @@ get_compile_command, get_hashes_from_ireq, is_pinned_requirement, + is_url_requirement, name_from_req, ) @@ -28,6 +29,11 @@ def test_format_requirement(from_line): assert format_requirement(ireq) == "test==1.2" +def test_format_requirement_url(from_line): + ireq = from_line("https://example.com/example.zip") + assert format_requirement(ireq) == "https://example.com/example.zip" + + def test_format_requirement_editable_vcs(from_editable): ireq = from_editable("git+git://fake.org/x/y.git#egg=y") assert format_requirement(ireq) == "-e git+git://fake.org/x/y.git#egg=y" @@ -149,6 +155,8 @@ def test_get_hashes_from_ireq(from_line): ("django>1.8", False), ("django~=1.8", False), ("django==1.*", False), + ("file:///example.zip", False), + ("https://example.com/example.zip", False), ], ) def test_is_pinned_requirement(from_line, line, expected): @@ -161,6 +169,24 @@ def test_is_pinned_requirement_editable(from_editable): assert not is_pinned_requirement(ireq) +@mark.parametrize( + ("line", "expected"), + [ + ("django==1.8", False), + ("django", False), + ("file:///example.zip", True), + ("https://example.com/example.zip", True), + ("https://example.com/example.zip#egg=example", True), + ("git+git://github.com/jazzband/pip-tools@master", True), + ("../example.zip", True), + ("/example.zip", True), + ], +) +def test_is_url_requirement(from_line, line, expected): + ireq = from_line(line) + assert is_url_requirement(ireq) is expected + + def test_name_from_req(from_line): ireq = from_line("django==1.8") assert name_from_req(ireq.req) == "django"