Support PEP-658 (#139)

* Support PEP 658 * update test * lint * changelog and doc fix * Unzip test file * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix changelog * gather requirements first when metadata is available * Update test * fix concurrency [integration] * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [integration] * Update mousebender * Update micropip/wheelinfo.py Co-authored-by: Agriya Khetarpal <[email protected]> * Fix name * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix tests * Fix another test * Update CHANGELOG.md --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Agriya Khetarpal <[email protected]>
pyodide · Dec 15, 2024 · 902c495 · 902c495
1 parent f02e8b9
commit 902c495
Show file tree

Hide file tree

Showing 11 changed files with 441 additions and 25 deletions.
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -12,6 +12,6 @@ python:
       path: .
 
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.12"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [0.8.0] - 2024/12/15
+
+### Added
+
+- Added support for PEP-658.
+  [#139](https://github.com/pyodide/micropip/pull/139)
+
 ## [0.7.2] - 2024/11/26
 
 ### Fixed

diff --git a/micropip/externals/mousebender/simple.py b/micropip/externals/mousebender/simple.py
@@ -47,6 +47,8 @@ class UnsupportedMIMEType(Exception):
         "dist-info-metadata": Union[bool, _HashesDict],
         "gpg-sig": bool,
         "yanked": Union[bool, str],
+        # PEP-714
+        "core-metadata": Union[bool, _HashesDict],
     },
     total=False,
 )
@@ -69,6 +71,8 @@ class ProjectFileDetails_1_0(_OptionalProjectFileDetails_1_0):
         "yanked": Union[bool, str],
         # PEP 700
         "upload-time": str,
+        # PEP 714
+        "core-metadata": Union[bool, _HashesDict],
     },
     total=False,
 )
@@ -173,8 +177,14 @@ def handle_starttag(
         # PEP 658:
         # ... each anchor tag pointing to a distribution MAY have a
         # data-dist-info-metadata attribute.
-        if "data-dist-info-metadata" in attrs:
-            found_metadata = attrs.get("data-dist-info-metadata")
+        # PEP 714:
+        # Clients consuming any of the HTML representations of the Simple API
+        # MUST read the PEP 658 metadata from the key data-core-metadata if it
+        # is present. They MAY optionally use the legacy data-dist-info-metadata
+        # if it is present but data-core-metadata is not.
+        metadata_fields = ["data-core-metadata", "data-dist-info-metadata"]
+        if any((metadata_field := field) in attrs for field in metadata_fields):
+            found_metadata = attrs.get(metadata_field)
             if found_metadata and found_metadata != "true":
                 # The repository SHOULD provide the hash of the Core Metadata
                 # file as the data-dist-info-metadata attribute's value using
@@ -212,12 +222,16 @@ def from_project_details_html(html: str, name: str) -> ProjectDetails_1_0:
         if "metadata" in archive_link:
             algorithm, value = archive_link["metadata"]
             if algorithm:
-                details["dist-info-metadata"] = {algorithm: value}
+                value = {algorithm: value}
             else:
-                details["dist-info-metadata"] = True
+                value = True
+
+            for key in ["core-metadata", "dist-info-metadata"]:
+                details[key] = value  # type: ignore[literal-required]
+
         for key in {"requires-python", "yanked", "gpg-sig"}:
             if key in archive_link:
-                details[key] = archive_link[key]  # type: ignore
+                details[key] = archive_link[key]  # type: ignore[literal-required]
         files.append(details)
     return {
         "meta": {"api-version": "1.0"},

diff --git a/micropip/package_index.py b/micropip/package_index.py
@@ -15,6 +15,7 @@
 from ._compat import HttpStatusError, fetch_string_and_headers
 from ._utils import is_package_compatible, parse_version
 from .externals.mousebender.simple import from_project_details_html
+from .types import DistributionMetadata
 from .wheelinfo import WheelInfo
 
 PYPI = "PYPI"
@@ -153,6 +154,11 @@ def _compatible_wheels(
             hashes = file["digests"] if "digests" in file else file["hashes"]
             sha256 = hashes.get("sha256")
 
+            # Check if the metadata file is available (PEP 658 / PEP-714)
+            core_metadata: DistributionMetadata = file.get("core-metadata") or file.get(
+                "data-dist-info-metadata"
+            )
+
             # Size of the file in bytes, if available (PEP 700)
             # This key is not available in the Simple API HTML response, so this field may be None
             size = file.get("size")
@@ -163,6 +169,7 @@ def _compatible_wheels(
                 version=version,
                 sha256=sha256,
                 size=size,
+                core_metadata=core_metadata,
             )
 
     @classmethod

diff --git a/micropip/transaction.py b/micropip/transaction.py
@@ -247,9 +247,28 @@ async def add_wheel(
         logger.info("Collecting %s%s", wheel.name, specifier)
         logger.info("  Downloading %s", wheel.url.split("/")[-1])
 
-        await wheel.download(self.fetch_kwargs)
+        wheel_download_task = asyncio.create_task(wheel.download(self.fetch_kwargs))
         if self.deps:
-            await self.gather_requirements(wheel.requires(extras))
+            # Case 1) If metadata file is available,
+            #         we can gather requirements without waiting for the wheel to be downloaded.
+            if wheel.pep658_metadata_available():
+                try:
+                    await wheel.download_pep658_metadata(self.fetch_kwargs)
+                except OSError:
+                    # If something goes wrong while downloading the metadata,
+                    # we have to wait for the wheel to be downloaded.
+                    await wheel_download_task
+
+                await asyncio.gather(
+                    self.gather_requirements(wheel.requires(extras)),
+                    wheel_download_task,
+                )
+
+            # Case 2) If metadata file is not available,
+            #         we have to wait for the wheel to be downloaded.
+            else:
+                await wheel_download_task
+                await self.gather_requirements(wheel.requires(extras))
 
         self.wheels.append(wheel)
 

diff --git a/micropip/types.py b/micropip/types.py
@@ -0,0 +1,5 @@
+# Distribution Metadata type (PEP 658)
+# None = metadata not available
+# bool = metadata available, but no checksum
+# dict[str, str] = metadata available with checksum
+DistributionMetadata = bool | dict[str, str] | None
diff --git a/micropip/wheelinfo.py b/micropip/wheelinfo.py
@@ -19,6 +19,7 @@
 )
 from ._utils import parse_wheel_filename
 from .metadata import Metadata, safe_name, wheel_dist_info_dir
+from .types import DistributionMetadata
 
 
 @dataclass
@@ -43,6 +44,7 @@ class WheelInfo:
     parsed_url: ParseResult
     sha256: str | None = None
     size: int | None = None  # Size in bytes, if available (PEP 700)
+    core_metadata: DistributionMetadata = None  # Wheel's metadata (PEP 658 / PEP-714)
 
     # Fields below are only available after downloading the wheel, i.e. after calling `download()`.
 
@@ -58,6 +60,7 @@ def __post_init__(self):
             self.url.startwith(p) for p in ("http:", "https:", "emfs:", "file:")
         ), self.url
         self._project_name = safe_name(self.name)
+        self.metadata_url = self.url + ".metadata"
 
     @classmethod
     def from_url(cls, url: str) -> "WheelInfo":
@@ -89,6 +92,7 @@ def from_package_index(
         version: Version,
         sha256: str | None,
         size: int | None,
+        core_metadata: DistributionMetadata = None,
     ) -> "WheelInfo":
         """Extract available metadata from response received from package index"""
         parsed_url = urlparse(url)
@@ -104,6 +108,7 @@ def from_package_index(
             parsed_url=parsed_url,
             sha256=sha256,
             size=size,
+            core_metadata=core_metadata,
         )
 
     async def install(self, target: Path) -> None:
@@ -130,10 +135,42 @@ async def download(self, fetch_kwargs: dict[str, Any]):
         if self._data is not None:
             return
 
-        self._data = await self._fetch_bytes(fetch_kwargs)
-        with zipfile.ZipFile(io.BytesIO(self._data)) as zf:
-            metadata_path = wheel_dist_info_dir(zf, self.name) + "/" + Metadata.PKG_INFO
-            self._metadata = Metadata(zipfile.Path(zf, metadata_path))
+        self._data = await self._fetch_bytes(self.url, fetch_kwargs)
+
+        # The wheel's metadata might be downloaded separately from the wheel itself.
+        # If it is not downloaded yet or if the metadata is not available, extract it from the wheel.
+        if self._metadata is None:
+            with zipfile.ZipFile(io.BytesIO(self._data)) as zf:
+                metadata_path = (
+                    Path(wheel_dist_info_dir(zf, self.name)) / Metadata.PKG_INFO
+                )
+                self._metadata = Metadata(zipfile.Path(zf, str(metadata_path)))
+
+    def pep658_metadata_available(self) -> bool:
+        """
+        Check if the wheel's metadata is exposed via PEP 658.
+        """
+        return self.core_metadata is not None
+
+    async def download_pep658_metadata(
+        self,
+        fetch_kwargs: dict[str, Any],
+    ) -> None:
+        """
+        Download the wheel's metadata. If the metadata is not available, return None.
+        """
+        if self.core_metadata is None:
+            return None
+
+        data = await self._fetch_bytes(self.metadata_url, fetch_kwargs)
+
+        match self.core_metadata:
+            case {"sha256": checksum}:  # sha256 checksum available
+                _validate_sha256_checksum(data, checksum)
+            case _:  # no checksum available
+                pass
+
+        self._metadata = Metadata(data)
 
     def requires(self, extras: set[str]) -> list[Requirement]:
         """
@@ -148,14 +185,14 @@ def requires(self, extras: set[str]) -> list[Requirement]:
         self._requires = requires
         return requires
 
-    async def _fetch_bytes(self, fetch_kwargs: dict[str, Any]):
+    async def _fetch_bytes(self, url: str, fetch_kwargs: dict[str, Any]):
         if self.parsed_url.scheme not in ("https", "http", "emfs", "file"):
             # Don't raise ValueError it gets swallowed
             raise TypeError(
-                f"Cannot download from a non-remote location: {self.url!r} ({self.parsed_url!r})"
+                f"Cannot download from a non-remote location: {url!r} ({self.parsed_url!r})"
             )
         try:
-            bytes = await fetch_bytes(self.url, fetch_kwargs)
+            bytes = await fetch_bytes(url, fetch_kwargs)
             return bytes
         except OSError as e:
             if self.parsed_url.hostname in [
@@ -165,7 +202,7 @@ async def _fetch_bytes(self, fetch_kwargs: dict[str, Any]):
                 raise e
             else:
                 raise ValueError(
-                    f"Can't fetch wheel from '{self.url}'. "
+                    f"Can't fetch wheel from {url!r}. "
                     "One common reason for this is when the server blocks "
                     "Cross-Origin Resource Sharing (CORS). "
                     "Check if the server is sending the correct 'Access-Control-Allow-Origin' header."

diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,7 @@ authors = [
 description = "A lightweight Python package installer for the web"
 readme = "README.md"
 license = { file="LICENSE" }
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
@@ -65,7 +65,7 @@ known-first-party = [
 ]
 
 [tool.mypy]
-python_version = "3.11"
+python_version = "3.12"
 show_error_codes = true
 warn_unreachable = true
 ignore_missing_imports = true
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -126,18 +126,24 @@ def __enter__(self):
     def __exit__(self, *args: Any):
         self._httpserver.__exit__(*args)
 
-    def _register_handler(self, path: Path) -> str:
-        self._httpserver.expect_request(f"/{path.name}").respond_with_data(
-            path.read_bytes(),
+    def _register_handler(self, endpoint: str, data: bytes) -> str:
+        self._httpserver.expect_request(f"/{endpoint}").respond_with_data(
+            data,
             content_type="application/zip",
             headers={"Access-Control-Allow-Origin": "*"},
         )
 
-        return self._httpserver.url_for(f"/{path.name}")
+        return self._httpserver.url_for(f"/{endpoint}")
 
     def add_wheel(self, path: Path, replace: bool = True):
         name, version = parse_wheel_filename(path.name)[0:2]
-        url = self._register_handler(path)
+        url = self._register_handler(path.name, path.read_bytes())
+
+        metadata_file_endpoint = path.with_suffix(".whl.metadata")
+        if metadata_file_endpoint.exists():
+            self._register_handler(
+                metadata_file_endpoint.name, metadata_file_endpoint.read_bytes()
+            )
 
         if name in self._wheels and not replace:
             return