Avoid reordering of equivalent packages from multiple fetchers (#762)

`Iterator.iter()` iterates over the set of links returned from `Crawler.crawl()`. This means the hash seed randomization potentially changes the order, when there are two packages equivalent in both version, package type (wheel, etc), platform tags, and they're both remote.

Ideally, these would not be any different, so it's not really a correctness issue, but I am using pex's resolver internals to store the actual URLs used, and they flap over time between different-but-equivalent ones.

Author: clintharrison

pex/sorter.py

@@ -37,7 +37,9 @@ def package_precedence(cls, package, precedence=DEFAULT_PACKAGE_PRECEDENCE):
         package.version,  # highest version
         cls.package_type_precedence(package, precedence=precedence),  # type preference
         cls.package_platform_tag_precedence(package),  # platform preference
-        package.local)  # prefer not fetching over the wire
+        package.local,  # prefer not fetching over the wire
+        package.url,  # otherwise equivalent files should deterministically sort
+    )
 
   def __init__(self, precedence=None):
     self._precedence = precedence or self.DEFAULT_PACKAGE_PRECEDENCE

tests/test_resolvable.py

@@ -6,7 +6,7 @@
 import pex.third_party.pkg_resources as pkg_resources
 from pex.interpreter import PythonInterpreter
 from pex.iterator import Iterator
-from pex.package import Package, SourcePackage
+from pex.package import Package, SourcePackage, WheelPackage
 from pex.resolvable import (
     Resolvable,
     ResolvableDirectory,
@@ -121,3 +121,26 @@ def test_resolvable_is_constraint_getter_setter():
   assert req.is_constraint is False
   req.is_constraint = True
   assert req.is_constraint is True
+
+
+def test_deterministic_packages_from_multiple_crawlers():
+  req = 'protobuf==3.9.1'
+  options_builder = ResolverOptionsBuilder(precedence=(WheelPackage,))
+  resolvable = ResolvableRequirement.from_string(req, options_builder)
+
+  pypi_source = Package.from_href('https://pypi.org/simple/protobuf/protobuf-3.9.1.tar.gz')
+  pypi_wheel = Package.from_href(
+    'https://pypi.org/simple/protobuf/protobuf-3.9.1-py2.py3-none-any.whl')
+  internal_wheel = Package.from_href(
+    'https://packages.internal.example/simple/protobuf/protobuf-3.9.1-py2.py3-none-any.whl')
+
+  # Multiple fetchers with equally-preferred packages should result in the same package every time,
+  # regardless the order that crawlers returned them.
+  mock_iterator = mock.create_autospec(Iterator, spec_set=True)
+  mock_iterator.iter.return_value = iter([pypi_source, pypi_wheel, internal_wheel])
+  url_order_one = [r.url for r in resolvable.compatible(mock_iterator)]
+
+  mock_iterator.iter.return_value = iter([pypi_source, internal_wheel, pypi_wheel])
+  url_order_two = [r.url for r in resolvable.compatible(mock_iterator)]
+
+  assert url_order_one == url_order_two

tests/test_sorter.py

@@ -28,7 +28,7 @@ def test_package_precedence():
   # overridden precedence
   PRECEDENCE = (EggPackage, WheelPackage)
   assert Sorter.package_precedence(source, PRECEDENCE) == (
-      source.version, -1, 0, True)  # unknown rank
+      source.version, -1, 0, True, source.url)  # unknown rank
   assert Sorter.package_precedence(whl, PRECEDENCE) > Sorter.package_precedence(
       source, PRECEDENCE)
   assert Sorter.package_precedence(egg, PRECEDENCE) > Sorter.package_precedence(