From 38d8eb09d7984bde9a81565bd9e1850317b452ba Mon Sep 17 00:00:00 2001 From: Kevin James Date: Thu, 10 Dec 2020 16:11:53 -0800 Subject: [PATCH] Add --no-backtracking option for new resolver Adds a new option `--no-backtracking` to the new resolver. Off by default (no change to default behaviour), enabling this option prevents us from running the backtracking behaviour when pinning fails and instead skip directly to the error reporting. This can be especially useful in a CI situation, where the developer is interested in knowing about conflicts up front in order to manually solve them, or is worried about allowing for a potentially large amount of time spent in backtracking. --- news/9258.feature.rst | 1 + src/pip/_internal/cli/cmdoptions.py | 11 +++++ src/pip/_internal/commands/download.py | 4 +- src/pip/_internal/commands/install.py | 4 +- src/pip/_internal/commands/wheel.py | 4 +- src/pip/_internal/resolution/base.py | 4 +- .../_internal/resolution/legacy/resolver.py | 4 +- .../resolution/resolvelib/resolver.py | 5 +- src/pip/_vendor/resolvelib/__init__.py | 2 +- src/pip/_vendor/resolvelib/resolvers.py | 12 +++-- src/pip/_vendor/vendor.txt | 2 +- tests/functional/test_install.py | 49 +++++++++++++++++++ tests/unit/test_req.py | 8 ++- 13 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 news/9258.feature.rst diff --git a/news/9258.feature.rst b/news/9258.feature.rst new file mode 100644 index 00000000000..b5907ae55a4 --- /dev/null +++ b/news/9258.feature.rst @@ -0,0 +1 @@ +New resolver: Add ``--no-backtracking`` option to enable failfast debugging. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 3543ed48bb3..3811318b4f0 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -927,6 +927,17 @@ def check_list_path_option(options): ), ) # type: Callable[..., Option] +disable_backtracking = partial( + Option, + '--no-backtracking', + dest='disable_backtracking', + action='store_true', + default=False, + help='Do not attempt to backtrack to resolve package conflicts. This will ' + 'cause conflicts which may be otherwise automatically solveable to ' + 'fail fast and require manual intervention!', +) # type: Callable[..., Option] + ########## # groups # diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 7405870aefc..829ce1dc59a 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -56,6 +56,7 @@ def add_options(self): self.cmd_opts.add_option(cmdoptions.no_build_isolation()) self.cmd_opts.add_option(cmdoptions.use_pep517()) self.cmd_opts.add_option(cmdoptions.no_use_pep517()) + self.cmd_opts.add_option(cmdoptions.disable_backtracking()) self.cmd_opts.add_option( '-d', '--dest', '--destination-dir', '--destination-directory', @@ -128,7 +129,8 @@ def run(self, options, args): self.trace_basic_info(finder) requirement_set = resolver.resolve( - reqs, check_supported_wheels=True + reqs, check_supported_wheels=True, + should_backtrack=not options.disable_backtracking ) downloaded = [] # type: List[str] diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index a4e10f260a2..3dc2a5cb186 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -216,6 +216,7 @@ def add_options(self): self.cmd_opts.add_option(cmdoptions.prefer_binary()) self.cmd_opts.add_option(cmdoptions.require_hashes()) self.cmd_opts.add_option(cmdoptions.progress_bar()) + self.cmd_opts.add_option(cmdoptions.disable_backtracking()) index_opts = cmdoptions.make_option_group( cmdoptions.index_group, @@ -318,7 +319,8 @@ def run(self, options, args): self.trace_basic_info(finder) requirement_set = resolver.resolve( - reqs, check_supported_wheels=not options.target_dir + reqs, check_supported_wheels=not options.target_dir, + should_backtrack=not options.disable_backtracking ) try: diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 39fd2bf8128..1c125c1ed58 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -80,6 +80,7 @@ def add_options(self): self.cmd_opts.add_option(cmdoptions.no_deps()) self.cmd_opts.add_option(cmdoptions.build_dir()) self.cmd_opts.add_option(cmdoptions.progress_bar()) + self.cmd_opts.add_option(cmdoptions.disable_backtracking()) self.cmd_opts.add_option( '--global-option', @@ -152,7 +153,8 @@ def run(self, options, args): self.trace_basic_info(finder) requirement_set = resolver.resolve( - reqs, check_supported_wheels=True + reqs, check_supported_wheels=True, + should_backtrack=not options.disable_backtracking ) reqs_to_build = [] # type: List[InstallRequirement] diff --git a/src/pip/_internal/resolution/base.py b/src/pip/_internal/resolution/base.py index 6d50555e531..b570d9e7a63 100644 --- a/src/pip/_internal/resolution/base.py +++ b/src/pip/_internal/resolution/base.py @@ -12,8 +12,8 @@ class BaseResolver(object): - def resolve(self, root_reqs, check_supported_wheels): - # type: (List[InstallRequirement], bool) -> RequirementSet + def resolve(self, root_reqs, check_supported_wheels, should_backtrack): + # type: (List[InstallRequirement], bool, bool) -> RequirementSet raise NotImplementedError() def get_installation_order(self, req_set): diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index d0fc1a7b316..88445b41a3b 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -149,8 +149,8 @@ def __init__( self._discovered_dependencies = \ defaultdict(list) # type: DiscoveredDependencies - def resolve(self, root_reqs, check_supported_wheels): - # type: (List[InstallRequirement], bool) -> RequirementSet + def resolve(self, root_reqs, check_supported_wheels, _should_backtrack): + # type: (List[InstallRequirement], bool, bool) -> RequirementSet """Resolve what operations need to be done As a side-effect of this method, the packages (and their dependencies) diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 30b860f6c48..a3cc7abb4b3 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -75,8 +75,8 @@ def __init__( self.upgrade_strategy = upgrade_strategy self._result = None # type: Optional[Result] - def resolve(self, root_reqs, check_supported_wheels): - # type: (List[InstallRequirement], bool) -> RequirementSet + def resolve(self, root_reqs, check_supported_wheels, should_backtrack): + # type: (List[InstallRequirement], bool, bool) -> RequirementSet constraints = {} # type: Dict[str, Constraint] user_requested = set() # type: Set[str] @@ -120,6 +120,7 @@ def resolve(self, root_reqs, check_supported_wheels): try_to_avoid_resolution_too_deep = 2000000 self._result = resolver.resolve( requirements, max_rounds=try_to_avoid_resolution_too_deep, + should_backtrack=should_backtrack, ) except ResolutionImpossible as e: diff --git a/src/pip/_vendor/resolvelib/__init__.py b/src/pip/_vendor/resolvelib/__init__.py index 5a400f23ed1..12f59c66500 100644 --- a/src/pip/_vendor/resolvelib/__init__.py +++ b/src/pip/_vendor/resolvelib/__init__.py @@ -11,7 +11,7 @@ "ResolutionTooDeep", ] -__version__ = "0.5.3" +__version__ = "0.5.4.dev0" from .providers import AbstractProvider, AbstractResolver diff --git a/src/pip/_vendor/resolvelib/resolvers.py b/src/pip/_vendor/resolvelib/resolvers.py index acf0f8a6b43..3ddb6939272 100644 --- a/src/pip/_vendor/resolvelib/resolvers.py +++ b/src/pip/_vendor/resolvelib/resolvers.py @@ -297,7 +297,7 @@ def _backtrack(self): # No way to backtrack anymore. return False - def resolve(self, requirements, max_rounds): + def resolve(self, requirements, max_rounds, should_backtrack): if self._states: raise RuntimeError("already resolved") @@ -341,7 +341,7 @@ def resolve(self, requirements, max_rounds): if failure_causes: # Backtrack if pinning fails. The backtrack process puts us in # an unpinned state, so we can work on it in the next round. - success = self._backtrack() + success = self._backtrack() if should_backtrack else False # Dead ends everywhere. Give up. if not success: @@ -413,7 +413,7 @@ class Resolver(AbstractResolver): base_exception = ResolverException - def resolve(self, requirements, max_rounds=100): + def resolve(self, requirements, max_rounds=100, should_backtrack=True): """Take a collection of constraints, spit out the resolution result. The return value is a representation to the final resolution result. It @@ -442,5 +442,9 @@ def resolve(self, requirements, max_rounds=100): `max_rounds` argument. """ resolution = Resolution(self.provider, self.reporter) - state = resolution.resolve(requirements, max_rounds=max_rounds) + state = resolution.resolve( + requirements, + max_rounds=max_rounds, + should_backtrack=should_backtrack, + ) return _build_result(state) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index c7bc37c16c4..24738dff8a1 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -16,7 +16,7 @@ requests==2.25.0 chardet==3.0.4 idna==2.10 urllib3==1.26.2 -resolvelib==0.5.3 +git+https://github.com/thekevjames/resolvelib.git@add-no-backtracking-option retrying==1.3.3 setuptools==44.0.0 six==1.15.0 diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index f9a807bca79..55aaeadd331 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1726,6 +1726,55 @@ def test_install_conflict_warning_can_be_suppressed(script, data): assert "Successfully installed pkgB-2.0" in result2.stdout, str(result2) +def test_install_incompatiblity_can_failfast(script, data): + # Build the following package dependency graph, which is impossible to + # resolve: + # + # /- pkgB == 1.0 -> pkgD == 1.0 + # / - pkgB == 1.1 -> pkgD == 1.0 + # pkgA==1.0 - + # \ - pkgC == 1.0 -> pkgD == 1.1 + # \- pkgC == 1.1 -> pkgD == 1.1 + pkgA_path = create_basic_wheel_for_package( + script, + name='pkgA', version='1.0', depends=[ + 'pkgb > 1.0, < 2.0', + 'pkgc > 1.0, < 2.0', + ], + ) + for minor in {0, 1}: + create_basic_wheel_for_package( + script, + name='pkgB', version='1.%s' % minor, + depends=['pkgd == 1.0'], + ) + create_basic_wheel_for_package( + script, + name='pkgC', version='1.%s' % minor, + depends=['pkgd == 1.1'], + ) + create_basic_wheel_for_package( + script, + name='pkgD', version='1.%s' % minor, + ) + + # Installing normally should run backtracking... + result1 = script.pip( + 'install', '--no-index', '--find-links', script.scratch_path, + pkgA_path, expect_error=True + ) + assert 'pip is looking at multiple versions of pkg' in result1.stdout + assert 'ResolutionImpossible' in result1.stderr + + # ...but the ``--no-backtracking`` flag will prevent this behaviour + result2 = script.pip( + 'install', '--no-index', '--find-links', script.scratch_path, + '--no-backtracking', pkgA_path, expect_error=True + ) + assert 'pip is looking at multiple versions of pkg' not in result2.stdout + assert 'ResolutionImpossible' in result2.stderr + + def test_target_install_ignores_distutils_config_install_prefix(script): prefix = script.scratch_path / 'prefix' distutils_config = Path(os.path.expanduser('~'), diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index e168a3cc164..ddfd4490639 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -121,6 +121,7 @@ def test_no_reuse_existing_build_dir(self, data): resolver.resolve, reqset.all_requirements, True, + True, ) # TODO: Update test when Python 2.7 is dropped. @@ -137,7 +138,7 @@ def test_environment_marker_extras(self, data): reqset.add_requirement(req) finder = make_test_finder(find_links=[data.find_links]) with self._basic_resolver(finder) as resolver: - reqset = resolver.resolve(reqset.all_requirements, True) + reqset = resolver.resolve(reqset.all_requirements, True, True) # This is hacky but does test both case in py2 and py3 if sys.version_info[:2] == (2, 7): assert reqset.has_requirement('simple') @@ -165,6 +166,7 @@ def test_missing_hash_with_require_hashes(self, data): resolver.resolve, reqset.all_requirements, True, + True, ) def test_missing_hash_with_require_hashes_in_reqs_file(self, data, tmpdir): @@ -217,6 +219,7 @@ def test_unsupported_hashes(self, data): resolver.resolve, reqset.all_requirements, True, + True, ) def test_unpinned_hash_checking(self, data): @@ -246,6 +249,7 @@ def test_unpinned_hash_checking(self, data): resolver.resolve, reqset.all_requirements, True, + True, ) def test_hash_mismatch(self, data): @@ -268,6 +272,7 @@ def test_hash_mismatch(self, data): resolver.resolve, reqset.all_requirements, True, + True, ) def test_unhashed_deps_on_require_hashes(self, data): @@ -291,6 +296,7 @@ def test_unhashed_deps_on_require_hashes(self, data): resolver.resolve, reqset.all_requirements, True, + True, ) def test_hashed_deps_on_require_hashes(self):