diff --git a/piptools/_compat/__init__.py b/piptools/_compat/__init__.py index e835b1e5d..47c30813e 100644 --- a/piptools/_compat/__init__.py +++ b/piptools/_compat/__init__.py @@ -21,13 +21,8 @@ WheelCache, cmdoptions, get_installed_distributions, - get_requirement_tracker, - global_tempdir_manager, install_req_from_editable, install_req_from_line, - is_dir_url, - is_file_url, - is_vcs_url, normalize_path, parse_requirements, path_to_url, diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index f465c598e..55597f174 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -2,31 +2,12 @@ from __future__ import absolute_import import importlib -from contextlib import contextmanager import pip from pip._vendor.packaging.version import parse as parse_version PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split("."))) -try: - from pip._internal.req.req_tracker import RequirementTracker -except ImportError: - - @contextmanager - def RequirementTracker(): - yield - - -# Introduced in pip 20.0 -try: - from pip._internal.utils.temp_dir import global_tempdir_manager -except ImportError: - - @contextmanager - def global_tempdir_manager(): - yield - def do_import(module_path, subimport=None, old_path=None): old_path = old_path or module_path @@ -72,47 +53,6 @@ def do_import(module_path, subimport=None, old_path=None): Session = do_import("_vendor.requests.sessions", "Session") Resolver = do_import("legacy_resolve", "Resolver", old_path="resolve") WheelCache = do_import("cache", "WheelCache", old_path="wheel") +install_req_from_line = do_import("req.constructors", "install_req_from_line") +install_req_from_editable = do_import("req.constructors", "install_req_from_editable") normalize_path = do_import("utils.misc", "normalize_path", old_path="utils") - -# pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if PIP_VERSION < (18, 1): - install_req_from_line = InstallRequirement.from_line - install_req_from_editable = InstallRequirement.from_editable -else: - install_req_from_line = do_import("req.constructors", "install_req_from_line") - install_req_from_editable = do_import( - "req.constructors", "install_req_from_editable" - ) - - -def is_vcs_url(link): - if PIP_VERSION < (19, 3): - _is_vcs_url = do_import("download", "is_vcs_url") - return _is_vcs_url(link) - - return link.is_vcs - - -def is_file_url(link): - if PIP_VERSION < (19, 3): - _is_file_url = do_import("download", "is_file_url") - return _is_file_url(link) - - return link.is_file - - -def is_dir_url(link): - if PIP_VERSION < (19, 3): - _is_dir_url = do_import("download", "is_dir_url") - return _is_dir_url(link) - - return link.is_existing_dir() - - -def get_requirement_tracker(): - if PIP_VERSION[:2] <= (19, 3): - return RequirementTracker() - - from pip._internal.req import req_tracker - - return req_tracker.get_requirement_tracker() diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index d90a5ffbf..9b7407d7a 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -5,25 +5,21 @@ import hashlib import os from contextlib import contextmanager -from functools import partial from shutil import rmtree +from pip._internal.commands import create_command +from pip._internal.req.req_tracker import get_requirement_tracker +from pip._internal.utils.temp_dir import TempDirectory, global_tempdir_manager + from .._compat import ( FAVORITE_HASH, - PIP_VERSION, Link, PyPI, RequirementSet, - Resolver as PipResolver, TemporaryDirectory, Wheel, WheelCache, contextlib, - get_requirement_tracker, - global_tempdir_manager, - is_dir_url, - is_file_url, - is_vcs_url, normalize_path, path_to_url, url_to_path, @@ -32,7 +28,6 @@ from ..exceptions import NoCandidateFound from ..logging import log from ..utils import ( - create_install_command, fs_str, is_pinned_requirement, is_url_requirement, @@ -56,16 +51,17 @@ class PyPIRepository(BaseRepository): """ def __init__(self, pip_args, cache_dir, build_isolation=False): - self.build_isolation = build_isolation - # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, # and pre) are deferred to pip. - self.command = create_install_command() + self.command = create_command("install") self.options, _ = self.command.parse_args(pip_args) if self.options.cache_dir: self.options.cache_dir = normalize_path(self.options.cache_dir) + self.options.build_isolation = build_isolation + self.options.require_hashes = False + self.session = self.command._build_session(self.options) self.finder = self.command._build_package_finder( options=self.options, session=self.session @@ -135,123 +131,51 @@ def find_best_match(self, ireq, prereleases=None): if not matching_candidates: raise NoCandidateFound(ireq, all_candidates, self.finder) - if PIP_VERSION < (19, 1): - best_candidate = max( - matching_candidates, key=self.finder._candidate_sort_key - ) - elif PIP_VERSION < (19, 2): - evaluator = self.finder.candidate_evaluator - best_candidate = evaluator.get_best_candidate(matching_candidates) - elif PIP_VERSION < (19, 3): - evaluator = self.finder.make_candidate_evaluator(ireq.name) - best_candidate = evaluator.get_best_candidate(matching_candidates) - else: - evaluator = self.finder.make_candidate_evaluator(ireq.name) - best_candidate_result = evaluator.compute_best_candidate( - matching_candidates - ) - best_candidate = best_candidate_result.best_candidate - - if PIP_VERSION[:2] <= (19, 3): - best_candidate_name = best_candidate.project - else: - best_candidate_name = best_candidate.name + evaluator = self.finder.make_candidate_evaluator(ireq.name) + best_candidate_result = evaluator.compute_best_candidate(matching_candidates) + best_candidate = best_candidate_result.best_candidate # Turn the candidate into a pinned InstallRequirement return make_install_requirement( - best_candidate_name, + best_candidate.name, best_candidate.version, ireq.extras, constraint=ireq.constraint, ) def resolve_reqs(self, download_dir, ireq, wheel_cache): - results = None - - if PIP_VERSION < (10,): - reqset = RequirementSet( - self.build_dir, - self.source_dir, + with get_requirement_tracker() as req_tracker, TempDirectory( + kind="req-tracker" + ) as temp_dir: + preparer = self.command.make_requirement_preparer( + temp_build_dir=temp_dir, + options=self.options, + req_tracker=req_tracker, + session=self.session, + finder=self.finder, + use_user_site=False, download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, - session=self.session, - wheel_cache=wheel_cache, ) - results = reqset._prepare_file(self.finder, ireq) - else: - from pip._internal.operations.prepare import RequirementPreparer - - preparer_kwargs = { - "build_dir": self.build_dir, - "src_dir": self.source_dir, - "download_dir": download_dir, - "wheel_download_dir": self._wheel_download_dir, - "progress_bar": "off", - "build_isolation": self.build_isolation, - } - resolver_kwargs = { - "finder": self.finder, - "session": self.session, - "upgrade_strategy": "to-satisfy-only", - "force_reinstall": False, - "ignore_dependencies": False, - "ignore_requires_python": False, - "ignore_installed": True, - "use_user_site": False, - } - make_install_req_kwargs = {"isolated": False, "wheel_cache": wheel_cache} - - if PIP_VERSION < (19, 3): - resolver_kwargs.update(**make_install_req_kwargs) - else: - from pip._internal.req.constructors import install_req_from_req_string - make_install_req = partial( - install_req_from_req_string, **make_install_req_kwargs - ) - resolver_kwargs["make_install_req"] = make_install_req - - if PIP_VERSION >= (20,): - del resolver_kwargs["session"] - del preparer_kwargs["progress_bar"] - - resolver = None - preparer = None - - if PIP_VERSION[:2] <= (19, 3): - tmp_dir_cm = contextlib.nullcontext() - else: - from pip._internal.utils.temp_dir import TempDirectory - - tmp_dir_cm = TempDirectory(kind="req-tracker") - - with get_requirement_tracker() as req_tracker, tmp_dir_cm as temp_build_dir: - # Pip 18 uses a requirement tracker to prevent fork bombs - if req_tracker: - preparer_kwargs["req_tracker"] = req_tracker + reqset = RequirementSet() + ireq.is_direct = True + reqset.add_requirement(ireq) - if PIP_VERSION[:2] <= (19, 3): - preparer = RequirementPreparer(**preparer_kwargs) - else: - preparer = self.command.make_requirement_preparer( - temp_build_dir=temp_build_dir, - options=self.options, - req_tracker=req_tracker, - session=self.session, - finder=self.finder, - use_user_site=self.options.use_user_site, - ) - - resolver_kwargs["preparer"] = preparer - reqset = RequirementSet() - ireq.is_direct = True - reqset.add_requirement(ireq) - - resolver = PipResolver(**resolver_kwargs) - resolver.require_hashes = False - results = resolver._resolve_one(reqset, ireq) + resolver = self.command.make_resolver( + preparer=preparer, + finder=self.finder, + options=self.options, + wheel_cache=wheel_cache, + use_user_site=False, + ignore_installed=True, + ignore_requires_python=False, + force_reinstall=False, + upgrade_strategy="to-satisfy-only", + ) + results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() + reqset.cleanup_files() return set(results) @@ -276,7 +200,7 @@ def get_dependencies(self, ireq): # If a download_dir is passed, pip will unnecessarely # archive the entire source directory download_dir = None - elif ireq.link and is_vcs_url(ireq.link): + elif ireq.link and ireq.link.is_vcs: # No download_dir for VCS sources. This also works around pip # using git-checkout-index, which gets rid of the .git dir. download_dir = None @@ -301,9 +225,7 @@ def get_dependencies(self, ireq): else: del os.environ["PIP_REQ_TRACKER"] - # WheelCache.cleanup() introduced in pip==10.0.0 - if PIP_VERSION >= (10,): - wheel_cache.cleanup() + wheel_cache.cleanup() return self._dependencies_cache[ireq] def get_hashes(self, ireq): @@ -316,7 +238,7 @@ def get_hashes(self, ireq): if ireq.link: link = ireq.link - if is_vcs_url(link) or (is_file_url(link) and is_dir_url(link)): + if link.is_vcs or (link.is_file and link.is_existing_dir()): # Return empty set for unhashable requirements. # Unhashable logic modeled on pip's # RequirementPreparer.prepare_linked_requirement @@ -348,14 +270,8 @@ def get_hashes(self, ireq): log.debug(" {}".format(ireq.name)) - def get_candidate_link(candidate): - if PIP_VERSION < (19, 2): - return candidate.location - return candidate.link - return { - self._get_file_hash(get_candidate_link(candidate)) - for candidate in matching_candidates + self._get_file_hash(candidate.link) for candidate in matching_candidates } def _get_file_hash(self, link): @@ -423,7 +339,7 @@ def open_local_or_remote_file(link, session): """ url = link.url_without_fragment - if is_file_url(link): + if link.is_file: # Local URL local_path = url_to_path(url) if os.path.isdir(local_path): diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 5c1588e58..27062e9a4 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -6,6 +6,7 @@ import tempfile from click.utils import safecall +from pip._internal.commands import create_command from .. import click from .._compat import install_req_from_line, parse_requirements @@ -17,9 +18,7 @@ from ..resolver import Resolver from ..utils import ( UNSAFE_PACKAGES, - create_install_command, dedup, - get_trusted_hosts, is_pinned_requirement, key_from_ireq, ) @@ -29,7 +28,7 @@ DEFAULT_REQUIREMENTS_OUTPUT_FILE = "requirements.txt" # Get default values of the pip's options (including options from pip.conf). -install_command = create_install_command() +install_command = create_command("install") pip_defaults = install_command.parser.get_default_values() @@ -415,7 +414,7 @@ def cli( generate_hashes=generate_hashes, default_index_url=repository.DEFAULT_INDEX_URL, index_urls=repository.finder.index_urls, - trusted_hosts=get_trusted_hosts(repository.finder), + trusted_hosts=repository.finder.trusted_hosts, format_control=repository.finder.format_control, allow_unsafe=allow_unsafe, find_links=repository.finder.find_links, diff --git a/piptools/utils.py b/piptools/utils.py index 463027bb2..17c6e5a1d 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -9,7 +9,7 @@ from click.utils import LazyFile from six.moves import shlex_quote -from ._compat import PIP_VERSION, InstallCommand, install_req_from_line +from ._compat import install_req_from_line from .click import style UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"} @@ -369,25 +369,3 @@ def get_compile_command(click_ctx): ) return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args)) - - -def create_install_command(): - """ - Return an instance of InstallCommand. - """ - if PIP_VERSION < (19, 3): - return InstallCommand() - - from pip._internal.commands import create_command - - return create_command("install") - - -def get_trusted_hosts(finder): - """ - Returns an iterable of trusted hosts from a given finder. - """ - if PIP_VERSION < (19, 2): - return (host for _, host, _ in finder.secure_origins) - - return finder.trusted_hosts diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index ea6421346..fbcb9bab4 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -10,7 +10,7 @@ from .constants import MINIMAL_WHEELS_PATH, PACKAGES_PATH from .utils import invoke -from piptools._compat.pip_compat import PIP_VERSION, path_to_url +from piptools._compat.pip_compat import path_to_url from piptools.scripts.compile import cli @@ -553,7 +553,6 @@ def test_generate_hashes_verbose(pip_conf, runner): assert expected_verbose_text in out.stderr -@pytest.mark.skipif(PIP_VERSION < (9,), reason="needs pip 9 or greater") def test_filter_pip_markers(pip_conf, runner): """ Check that pip-compile works with pip environment markers (PEP496) diff --git a/tests/test_repository_pypi.py b/tests/test_repository_pypi.py index 45c4abad2..9bc270d88 100644 --- a/tests/test_repository_pypi.py +++ b/tests/test_repository_pypi.py @@ -3,8 +3,7 @@ import mock import pytest -from piptools._compat import PackageFinder -from piptools._compat.pip_compat import PIP_VERSION, Link, Session, path_to_url +from piptools._compat.pip_compat import Link, Session, path_to_url from piptools.repositories import PyPIRepository from piptools.repositories.pypi import open_local_or_remote_file @@ -126,48 +125,6 @@ def test_pypirepo_source_dir_is_str(pypi_repository): assert isinstance(pypi_repository.source_dir, str) -@pytest.mark.skipif( - PIP_VERSION >= (10,), - reason="RequirementSet objects don't take arguments after pip 10.", -) -def test_pypirepo_calls_reqset_with_str_paths(pypi_repository, from_line): - """ - Make sure that paths passed to RequirementSet init are str. - - Passing unicode paths on Python 2 could make pip fail later on - unpack, if the package contains non-ASCII file names, because - non-ASCII str and unicode paths cannot be combined. - """ - with mock.patch("piptools.repositories.pypi.RequirementSet") as mocked_init: - ireq = from_line("ansible==2.4.0.0") - - # Setup a mock object to be returned from the RequirementSet call - mocked_reqset = mock.MagicMock() - mocked_init.return_value = mocked_reqset - - # Do the call - pypi_repository.get_dependencies(ireq) - - # Check that RequirementSet init is called with correct type arguments - assert mocked_init.call_count == 1 - (init_call_args, init_call_kwargs) = mocked_init.call_args - assert isinstance(init_call_args[0], str) - assert isinstance(init_call_args[1], str) - assert isinstance(init_call_kwargs.get("download_dir"), str) - assert isinstance(init_call_kwargs.get("wheel_download_dir"), str) - - # Check that _prepare_file is called correctly - assert mocked_reqset._prepare_file.call_count == 1 - (pf_call_args, pf_call_kwargs) = mocked_reqset._prepare_file.call_args - (called_with_finder, called_with_ireq) = pf_call_args - assert isinstance(called_with_finder, PackageFinder) - assert called_with_ireq == ireq - assert not pf_call_kwargs - - -@pytest.mark.skipif( - PIP_VERSION < (10,), reason="WheelCache.cleanup() introduced in pip==10.0.0" -) @mock.patch("piptools.repositories.pypi.PyPIRepository.resolve_reqs") # to run offline @mock.patch("piptools.repositories.pypi.WheelCache") def test_wheel_cache_cleanup_called( diff --git a/tests/test_utils.py b/tests/test_utils.py index 99be592fd..e8af29213 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,19 +1,15 @@ # coding: utf-8 from __future__ import unicode_literals -import itertools import os -import pytest import six from pytest import mark, raises from six.moves import shlex_quote -from piptools.repositories import PyPIRepository from piptools.scripts.compile import cli as compile_cli from piptools.utils import ( as_tuple, - create_install_command, dedup, flat_map, force_text, @@ -22,7 +18,6 @@ fs_str, get_compile_command, get_hashes_from_ireq, - get_trusted_hosts, is_pinned_requirement, is_url_requirement, name_from_req, @@ -329,29 +324,3 @@ def test_get_compile_command_sort_args(tmpdir_cwd): "--no-annotate --no-emit-trusted-host --no-index " "requirements.in setup.py" ) - - -def test_create_install_command(): - """ - Test create_install_command returns an instance of InstallCommand. - """ - install_command = create_install_command() - assert install_command.name == "install" - - -@mark.parametrize( - "hosts", - [ - pytest.param((), id="no hosts"), - pytest.param(("example.com",), id="single host"), - pytest.param(("example.com:8080",), id="host with port"), - pytest.param(("example1.com", "example2.com:8080"), id="multiple hosts"), - ], -) -def test_get_trusted_hosts(hosts, tmpdir): - """ - Test get_trusted_hosts(finder) returns a list of hosts. - """ - pip_args = list(itertools.chain(*zip(["--trusted-host"] * len(hosts), hosts))) - repository = PyPIRepository(pip_args, cache_dir=str(tmpdir / "pypi-repo")) - assert tuple(get_trusted_hosts(repository.finder)) == hosts