diff --git a/pex/pip.py b/pex/pip.py index 4e3c62c98..beb9ca0c5 100644 --- a/pex/pip.py +++ b/pex/pip.py @@ -63,6 +63,7 @@ List, Mapping, Optional, + Pattern, Protocol, Sequence, Tuple, @@ -405,6 +406,7 @@ def __init__( ) self._analysis_completed = False self._locked_resolve = None # type: Optional[LockedResolve] + self._done_building_re = None # type: Optional[Pattern] def should_collect(self, returncode): # type: (int) -> bool @@ -429,20 +431,48 @@ def _extract_resolve_data(url): def analyze(self, line): # type: (str) -> _LogAnalyzer.Continue[None] + # The log sequence for processing a resolved requirement is as follows (log lines irrelevant + # to our purposes omitted): + # + # 1.) "... Found link ..." + # ... + # 1.) "... Found link ..." + # 2.) "... Added from ... to build tracker ..." + # 3.) Lines related to extracting metadata from 's artifact + # 4.) "... Removed from ... from build tracker ..." + # + # The lines in section 3 can contain this same pattern of lines if the metadata extraction + # proceeds via PEP-517 which recursively uses Pip to resolve build dependencies. We want to + # ignore this recursion since a lock should only contain install requirements and not build + # requirements (If a build proceeds differently tomorrow than today then we don't care as + # long as the final built artifact hashes the same. In other words, we completely rely on a + # cryptographic fingerprint for reproducibility and security guarantees from a lock). + if self._analysis_completed: raise self.StateError("Line analysis was requested after the log analysis completed.") + if self._done_building_re: + if self._done_building_re.search(line): + self._done_building_re = None + return self.Continue() + match = re.search( - r"Added (?P.*) from (?P[^\s]+) (\(from (?P.*)\) )?to build " + r"Added (?P.*) from (?P[^\s]+) (?:\(from (?P.*)\) )?to build " r"tracker", line, ) if match: - requirement = Requirement.parse(match.group("requirement")) - project_name_and_version, partial_artifact = self._extract_resolve_data( - match.group("url") + raw_requirement = match.group("requirement") + url = match.group("url") + self._done_building_re = re.compile( + r"Removed {requirement} from {url} (?:.* )?from build tracker".format( + requirement=re.escape(raw_requirement), url=re.escape(url) + ) ) + requirement = Requirement.parse(raw_requirement) + project_name_and_version, partial_artifact = self._extract_resolve_data(url) + from_ = match.group("from") if from_: via = tuple(from_.split("->")) @@ -451,6 +481,7 @@ def analyze(self, line): additional_artifacts = self._links[project_name_and_version] additional_artifacts.discard(partial_artifact) + self._links.clear() self._resolved_requirements.append( ResolvedRequirement( diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py index 557251eee..0cddae8ba 100644 --- a/tests/integration/cli/commands/test_lock.py +++ b/tests/integration/cli/commands/test_lock.py @@ -530,3 +530,86 @@ def test_update_partial(tmpdir): ) result.assert_success() assert DUAL_UPDATE_LOCKFILE == lockfile.load(lock_file_path) + + +def test_excludes_pep517_build_requirements_issue_1565(tmpdir): + # type: (Any) -> None + + # Here we resolve ansicolors 1.0.2 and find 2020.12.3 which are both pure legacy sdist + # distributions that will need to download build requirements using Pip since we force PEP-517. + # The cowsay 4.0 requirement is satisfied by a universal wheel and has no build requirements as + # a result. + + result = run_pex3( + "lock", + "create", + "ansicolors==1.0.2", + "find==2020.12.3", + "cowsay==4.0", + "--force-pep517", + ) + result.assert_success() + lock = lockfile.loads(result.output) + + assert 1 == len(lock.locked_resolves) + assert ( + SortedTuple( + [ + LockedRequirement.create( + pin=Pin( + project_name=ProjectName(project_name="ansicolors"), + version=Version(version="1.0.2"), + ), + artifact=Artifact( + url=( + "https://files.pythonhosted.org/packages/ac/c1/" + "e21f0a1258ff927d124a72179669dcc7efcb57b22df8cd0e49ed8f1a308c/" + "ansicolors-1.0.2.tar.gz" + ), + fingerprint=Fingerprint( + algorithm="sha256", + hash="7664530bb992e3847b61e3aab1580b4df9ed00c5898e80194a9933bc9c80950a", + ), + ), + requirement=Requirement.parse("ansicolors==1.0.2"), + ), + LockedRequirement.create( + pin=Pin( + project_name=ProjectName(project_name="find"), + version=Version(version="2020.12.3"), + ), + artifact=Artifact( + url=( + "https://files.pythonhosted.org/packages/91/1c/" + "90cac4602ec146ce6f055b2e9598f46da08e941dd860f0498af764407b7e/" + "find-2020.12.3.tar.gz" + ), + fingerprint=Fingerprint( + algorithm="sha256", + hash="7dadadb63e13de019463f13d83e0e0567a963cad99a568d0f0001ac1104d8210", + ), + ), + requirement=Requirement.parse("find==2020.12.3"), + ), + LockedRequirement.create( + pin=Pin( + project_name=ProjectName(project_name="cowsay"), + version=Version(version="4"), + ), + artifact=Artifact( + url=( + "https://files.pythonhosted.org/packages/b7/65/" + "38f31ef16efc312562f68732098d6f7ba3b2c108a4aaa8ac8ba673ee0871/" + "cowsay-4.0-py2.py3-none-any.whl" + ), + fingerprint=Fingerprint( + algorithm="sha256", + hash="2594b11d6624fff4bf5147b6bdd510ada54a7b5b4e3f2b15ac2a6d3cf99e0bf8", + ), + ), + requirement=Requirement.parse("cowsay==4.0"), + ), + ] + ) + == lock.locked_resolves[0].locked_requirements + )