Merge pull request #672 from techalchemy/pip18-support

Pip18 support

.appveyor.yml

@@ -7,6 +7,8 @@ environment:
           PIP: 9.0.1
         - TOXENV: py27-pip9.0.3
           PIP: 9.0.3
+        - TOXENV: py27-pip10.0.1
+          PIP: 10.0.1
         - TOXENV: py27-pipmaster
           PIP: master
         - TOXENV: py27-piplatest
@@ -17,6 +19,8 @@ environment:
           PIP: 9.0.1
         - TOXENV: py34-pip9.0.3
           PIP: 9.0.3
+        - TOXENV: py34-pip10.0.1
+          PIP: 10.0.1
         - TOXENV: py34-pipmaster
           PIP: master
         - TOXENV: py34-piplatest
@@ -27,6 +31,8 @@ environment:
           PIP: 9.0.1
         - TOXENV: py35-pip9.0.3
           PIP: 9.0.3
+        - TOXENV: py35-pip10.0.1
+          PIP: 10.0.1
         - TOXENV: py35-pipmaster
           PIP: master
         - TOXENV: py35-piplatest
@@ -37,10 +43,24 @@ environment:
           PIP: 9.0.1
         - TOXENV: py36-pip9.0.3
           PIP: 9.0.3
+        - TOXENV: py36-pip10.0.1
+          PIP: 10.0.1
         - TOXENV: py36-pipmaster
           PIP: master
         - TOXENV: py36-piplatest
           PIP: latest
+        - TOXENV: py37-pip8.1.1
+          PIP: 8.1.1
+        - TOXENV: py37-pip9.0.1
+          PIP: 9.0.1
+        - TOXENV: py37-pip9.0.3
+          PIP: 9.0.3
+        - TOXENV: py37-pip10.0.1
+          PIP: 10.0.1
+        - TOXENV: py37-pipmaster
+          PIP: master
+        - TOXENV: py37-piplatest
+          PIP: latest
 
 install:
     - C:\Python36\python -m pip install tox

.travis.yml

@@ -11,6 +11,7 @@ env:
   - PIP=8.1.1
   - PIP=9.0.1
   - PIP=9.0.3
+  - PIP=10.0.1
   - PIP=master
   - PIP=latest
 

piptools/_compat/pip_compat.py

@@ -2,17 +2,21 @@
 import importlib
 
 def do_import(module_path, subimport=None, old_path=None):
-    internal = 'pip._internal.{0}'.format(module_path)
     old_path = old_path or module_path
-    pip9 = 'pip.{0}'.format(old_path)
-    try:
-        _tmp = importlib.import_module(internal)
-    except ImportError:
-        _tmp = importlib.import_module(pip9)
-    if subimport:
-        return getattr(_tmp, subimport, _tmp)
-    return _tmp
-
+    prefixes = ["pip._internal", "pip"]
+    paths = [module_path, old_path]
+    search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None]
+    package = subimport if subimport else None
+    for to_import in search_order:
+        if not subimport:
+            to_import, _, package = to_import.rpartition(".")
+        try:
+            imported = importlib.import_module(to_import)
+        except ImportError:
+            continue
+        else:
+            return getattr(imported, package)
+
 
 InstallRequirement = do_import('req.req_install', 'InstallRequirement')
 parse_requirements = do_import('req.req_file', 'parse_requirements')
@@ -24,7 +28,7 @@ def do_import(module_path, subimport=None, old_path=None):
 PackageFinder = do_import('index', 'PackageFinder')
 FormatControl = do_import('index', 'FormatControl')
 Wheel = do_import('wheel', 'Wheel')
-Command = do_import('basecommand', 'Command')
-cmdoptions = do_import('cmdoptions')
+Command = do_import('cli.base_command', 'Command', old_path='basecommand')
+cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions')
 get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils')
 PyPI = do_import('models.index', 'PyPI')

piptools/repositories/pypi.py

@@ -24,12 +24,12 @@
                      make_install_requirement)
 from .base import BaseRepository
 
-
 try:
-    from pip._internal.operations.prepare import RequirementPreparer
-    from pip._internal.resolve import Resolver as PipResolver
+    from pip._internal.req.req_tracker import RequirementTracker
 except ImportError:
-    pass
+    @contextmanager
+    def RequirementTracker():
+        yield
 
 try:
     from pip._internal.cache import WheelCache
@@ -49,7 +49,6 @@ class PyPIRepository(BaseRepository):
     def __init__(self, pip_options, session):
         self.session = session
         self.pip_options = pip_options
-        self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control)
 
         index_urls = [pip_options.index_url] + pip_options.extra_index_urls
         if pip_options.no_index:
@@ -130,6 +129,62 @@ def find_best_match(self, ireq, prereleases=None):
             best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint
         )
 
+    def resolve_reqs(self, download_dir, ireq, wheel_cache):
+        results = None
+        try:
+            from pip._internal.operations.prepare import RequirementPreparer
+            from pip._internal.resolve import Resolver as PipResolver
+        except ImportError:
+            # Pip 9 and below
+            reqset = RequirementSet(
+                self.build_dir,
+                self.source_dir,
+                download_dir=download_dir,
+                wheel_download_dir=self._wheel_download_dir,
+                session=self.session,
+                wheel_cache=wheel_cache
+            )
+            results = reqset._prepare_file(self.finder, ireq)
+        else:
+            # pip >= 10
+            preparer_kwargs = {
+                'build_dir': self.build_dir,
+                'src_dir': self.source_dir,
+                'download_dir': download_dir,
+                'wheel_download_dir': self._wheel_download_dir,
+                'progress_bar': 'off',
+                'build_isolation': False
+            }
+            resolver_kwargs = {
+                'finder': self.finder,
+                'session': self.session,
+                'upgrade_strategy': "to-satisfy-only",
+                'force_reinstall': False,
+                'ignore_dependencies': False,
+                'ignore_requires_python': False,
+                'ignore_installed': True,
+                'isolated': False,
+                'wheel_cache': wheel_cache,
+                'use_user_site': False
+            }
+            resolver = None
+            preparer = None
+            with RequirementTracker() as req_tracker:
+                # Pip 18 uses a requirement tracker to prevent fork bombs
+                if req_tracker:
+                    preparer_kwargs['req_tracker'] = req_tracker
+                preparer = RequirementPreparer(**preparer_kwargs)
+                resolver_kwargs['preparer'] = preparer
+                reqset = RequirementSet()
+                ireq.is_direct = True
+                reqset.add_requirement(ireq)
+                resolver = PipResolver(**resolver_kwargs)
+                resolver.require_hashes = False
+                results = resolver._resolve_one(reqset, ireq)
+                reqset.cleanup_files()
+
+        return set(results)
+
     def get_dependencies(self, ireq):
         """
         Given a pinned or an editable InstallRequirement, returns a set of
@@ -156,50 +211,21 @@ def get_dependencies(self, ireq):
             if not os.path.isdir(self._wheel_download_dir):
                 os.makedirs(self._wheel_download_dir)
 
+            wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control)
+            prev_tracker = os.environ.get('PIP_REQ_TRACKER')
             try:
-                # Pip < 9 and below
-                reqset = RequirementSet(
-                    self.build_dir,
-                    self.source_dir,
-                    download_dir=download_dir,
-                    wheel_download_dir=self._wheel_download_dir,
-                    session=self.session,
-                    wheel_cache=self.wheel_cache,
-                )
-                self._dependencies_cache[ireq] = reqset._prepare_file(
-                    self.finder,
-                    ireq
-                )
-            except TypeError:
-                # Pip >= 10 (new resolver!)
-                preparer = RequirementPreparer(
-                    build_dir=self.build_dir,
-                    src_dir=self.source_dir,
-                    download_dir=download_dir,
-                    wheel_download_dir=self._wheel_download_dir,
-                    progress_bar='off',
-                    build_isolation=False
-                )
-                reqset = RequirementSet()
-                ireq.is_direct = True
-                reqset.add_requirement(ireq)
-                self.resolver = PipResolver(
-                    preparer=preparer,
-                    finder=self.finder,
-                    session=self.session,
-                    upgrade_strategy="to-satisfy-only",
-                    force_reinstall=False,
-                    ignore_dependencies=False,
-                    ignore_requires_python=False,
-                    ignore_installed=True,
-                    isolated=False,
-                    wheel_cache=self.wheel_cache,
-                    use_user_site=False,
-                )
-                self.resolver.resolve(reqset)
-                self._dependencies_cache[ireq] = reqset.requirements.values()
-            reqset.cleanup_files()
-        return set(self._dependencies_cache[ireq])
+                self._dependencies_cache[ireq] = self.resolve_reqs(download_dir, ireq, wheel_cache)
+            finally:
+                if 'PIP_REQ_TRACKER' in os.environ:
+                    if prev_tracker:
+                        os.environ['PIP_REQ_TRACKER'] = prev_tracker
+                    else:
+                        del os.environ['PIP_REQ_TRACKER']
+                try:
+                    self.wheel_cache.cleanup()
+                except AttributeError:
+                    pass
+        return self._dependencies_cache[ireq]
 
     def get_hashes(self, ireq):
         """

tests/test_repositories.py

@@ -1,8 +1,11 @@
 from mock import MagicMock, patch
+from pip._vendor.packaging.version import parse as parse_version
+from pip import __version__ as pip_version
 from piptools._compat import PackageFinder, InstallRequirement
 
 from piptools.repositories.pypi import PyPIRepository
 from piptools.scripts.compile import get_pip_command
+import pytest
 
 
 def test_pypirepo_build_dir_is_str():
@@ -13,6 +16,10 @@ def test_pypirepo_source_dir_is_str():
     assert isinstance(get_pypi_repository().source_dir, str)
 
 
+@pytest.mark.skipif(
+    parse_version(pip_version) >= parse_version('10.0.0'),
+    reason="RequirementSet objects don't take arguments after pip 10."
+)
 def test_pypirepo_calls_reqset_with_str_paths():
     """
     Make sure that paths passed to RequirementSet init are str.

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist =
-    py{27,34,35,36,py}-pip{8.1.1,9.0.1,9.0.3,latest,master}
+    py{27,34,35,36,37,py}-pip{8.1.1,9.0.1,9.0.3,10.0.1,latest,master}
     flake8
     readme
 skip_missing_interpreters = True
@@ -12,6 +12,7 @@ deps =
     pip8.1.1: pip==8.1.1
     pip9.0.1: pip==9.0.1
     pip9.0.3: pip==9.0.3
+    pip10.0.1: pip==10.0.1
     coverage
     mock
     pytest
@@ -21,6 +22,7 @@ setenv =
     pip8.1.1: PIP=8.1.1
     pip9.0.1: PIP=9.0.1
     pip9.0.3: PIP=9.0.3
+    pip10.0.1: PIP=10.0.1
 install_command= python -m pip install {opts} {packages}
 commands =
     pip --version
@@ -42,5 +44,6 @@ PIP =
     8.1.1: pip8.1.1
     9.0.1: pip9.0.1
     9.0.3: pip9.0.3
+    10.0.1: pip10.0.1
     latest: piplatest
     master: pipmaster