Fix locks to exclude build requirements. (#1566)

Previously build requirements resolved recursively via Pip PEP-517
handling would show up in locks.

Fixes #1565

pex/pip.py

@@ -63,6 +63,7 @@
         List,
         Mapping,
         Optional,
+        Pattern,
         Protocol,
         Sequence,
         Tuple,
@@ -405,6 +406,7 @@ def __init__(
         )
         self._analysis_completed = False
         self._locked_resolve = None  # type: Optional[LockedResolve]
+        self._done_building_re = None  # type: Optional[Pattern]
 
     def should_collect(self, returncode):
         # type: (int) -> bool
@@ -429,20 +431,48 @@ def _extract_resolve_data(url):
     def analyze(self, line):
         # type: (str) -> _LogAnalyzer.Continue[None]
 
+        # The log sequence for processing a resolved requirement is as follows (log lines irrelevant
+        # to our purposes omitted):
+        #
+        # 1.) "... Found link <url1> ..."
+        # ...
+        # 1.) "... Found link <urlN> ..."
+        # 2.) "... Added <requirement pin> from <url> ... to build tracker ..."
+        # 3.) Lines related to extracting metadata from <requirement pin>'s artifact
+        # 4.) "... Removed <requirement pin> from <url> ... from build tracker ..."
+        #
+        # The lines in section 3 can contain this same pattern of lines if the metadata extraction
+        # proceeds via PEP-517 which recursively uses Pip to resolve build dependencies. We want to
+        # ignore this recursion since a lock should only contain install requirements and not build
+        # requirements (If a build proceeds differently tomorrow than today then we don't care as
+        # long as the final built artifact hashes the same. In other words, we completely rely on a
+        # cryptographic fingerprint for reproducibility and security guarantees from a lock).
+
         if self._analysis_completed:
             raise self.StateError("Line analysis was requested after the log analysis completed.")
 
+        if self._done_building_re:
+            if self._done_building_re.search(line):
+                self._done_building_re = None
+            return self.Continue()
+
         match = re.search(
-            r"Added (?P<requirement>.*) from (?P<url>[^\s]+) (\(from (?P<from>.*)\) )?to build "
+            r"Added (?P<requirement>.*) from (?P<url>[^\s]+) (?:\(from (?P<from>.*)\) )?to build "
             r"tracker",
             line,
         )
         if match:
-            requirement = Requirement.parse(match.group("requirement"))
-            project_name_and_version, partial_artifact = self._extract_resolve_data(
-                match.group("url")
+            raw_requirement = match.group("requirement")
+            url = match.group("url")
+            self._done_building_re = re.compile(
+                r"Removed {requirement} from {url} (?:.* )?from build tracker".format(
+                    requirement=re.escape(raw_requirement), url=re.escape(url)
+                )
             )
 
+            requirement = Requirement.parse(raw_requirement)
+            project_name_and_version, partial_artifact = self._extract_resolve_data(url)
+
             from_ = match.group("from")
             if from_:
                 via = tuple(from_.split("->"))
@@ -451,6 +481,7 @@ def analyze(self, line):
 
             additional_artifacts = self._links[project_name_and_version]
             additional_artifacts.discard(partial_artifact)
+            self._links.clear()
 
             self._resolved_requirements.append(
                 ResolvedRequirement(

tests/integration/cli/commands/test_lock.py

@@ -530,3 +530,86 @@ def test_update_partial(tmpdir):
     )
     result.assert_success()
     assert DUAL_UPDATE_LOCKFILE == lockfile.load(lock_file_path)
+
+
+def test_excludes_pep517_build_requirements_issue_1565(tmpdir):
+    # type: (Any) -> None
+
+    # Here we resolve ansicolors 1.0.2 and find 2020.12.3 which are both pure legacy sdist
+    # distributions that will need to download build requirements using Pip since we force PEP-517.
+    # The cowsay 4.0 requirement is satisfied by a universal wheel and has no build requirements as
+    # a result.
+
+    result = run_pex3(
+        "lock",
+        "create",
+        "ansicolors==1.0.2",
+        "find==2020.12.3",
+        "cowsay==4.0",
+        "--force-pep517",
+    )
+    result.assert_success()
+    lock = lockfile.loads(result.output)
+
+    assert 1 == len(lock.locked_resolves)
+    assert (
+        SortedTuple(
+            [
+                LockedRequirement.create(
+                    pin=Pin(
+                        project_name=ProjectName(project_name="ansicolors"),
+                        version=Version(version="1.0.2"),
+                    ),
+                    artifact=Artifact(
+                        url=(
+                            "https://files.pythonhosted.org/packages/ac/c1/"
+                            "e21f0a1258ff927d124a72179669dcc7efcb57b22df8cd0e49ed8f1a308c/"
+                            "ansicolors-1.0.2.tar.gz"
+                        ),
+                        fingerprint=Fingerprint(
+                            algorithm="sha256",
+                            hash="7664530bb992e3847b61e3aab1580b4df9ed00c5898e80194a9933bc9c80950a",
+                        ),
+                    ),
+                    requirement=Requirement.parse("ansicolors==1.0.2"),
+                ),
+                LockedRequirement.create(
+                    pin=Pin(
+                        project_name=ProjectName(project_name="find"),
+                        version=Version(version="2020.12.3"),
+                    ),
+                    artifact=Artifact(
+                        url=(
+                            "https://files.pythonhosted.org/packages/91/1c/"
+                            "90cac4602ec146ce6f055b2e9598f46da08e941dd860f0498af764407b7e/"
+                            "find-2020.12.3.tar.gz"
+                        ),
+                        fingerprint=Fingerprint(
+                            algorithm="sha256",
+                            hash="7dadadb63e13de019463f13d83e0e0567a963cad99a568d0f0001ac1104d8210",
+                        ),
+                    ),
+                    requirement=Requirement.parse("find==2020.12.3"),
+                ),
+                LockedRequirement.create(
+                    pin=Pin(
+                        project_name=ProjectName(project_name="cowsay"),
+                        version=Version(version="4"),
+                    ),
+                    artifact=Artifact(
+                        url=(
+                            "https://files.pythonhosted.org/packages/b7/65/"
+                            "38f31ef16efc312562f68732098d6f7ba3b2c108a4aaa8ac8ba673ee0871/"
+                            "cowsay-4.0-py2.py3-none-any.whl"
+                        ),
+                        fingerprint=Fingerprint(
+                            algorithm="sha256",
+                            hash="2594b11d6624fff4bf5147b6bdd510ada54a7b5b4e3f2b15ac2a6d3cf99e0bf8",
+                        ),
+                    ),
+                    requirement=Requirement.parse("cowsay==4.0"),
+                ),
+            ]
+        )
+        == lock.locked_resolves[0].locked_requirements
+    )