Merge remote-tracking branch 'upstream/main' into compat-layer-unvendor

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Keep GitHub Actions up to date with GitHub's Dependabot...
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: monthly

.github/workflows/main.yml

@@ -9,6 +9,9 @@ concurrency:
   group: main-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: 3
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -19,21 +22,23 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        pyodide-version: ['0.27.0a2']
+        pyodide-version: ["0.27.2"]
         test-config: [
-          # FIXME: recent version of chrome gets timeout
-          {runner: selenium, runtime: chrome, runtime-version: "125" },
-          {runner: selenium, runtime: node, runtime-version: "22" },
-        ]
+            # FIXME: recent version of chrome gets timeout
+            { runner: selenium, runtime: chrome, runtime-version: "125" },
+            { runner: selenium, runtime: node, runtime-version: "22" },
+          ]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.12
 
-      - uses: pyodide/pyodide-actions/download-pyodide@v1
+      - uses: pyodide/pyodide-actions/download-pyodide@v2
         with:
           version: ${{ matrix.pyodide-version }}
           to: dist
@@ -73,7 +78,7 @@ jobs:
             --runner=${{ matrix.test-config.runner }} \
             --rt ${{ matrix.test-config.runtime }}
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v5
         if: ${{ github.event.repo.name == 'pyodide/micropip' || github.event_name == 'pull_request' }}
         with:
           fail_ci_if_error: false
@@ -85,9 +90,10 @@ jobs:
       run-integration-test: ${{ steps.check-integration-test-trigger.outputs.trigger }}
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@cbb722410c2e876e24abbe8de2cc27693e501dcb # v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
 
       - id: check-integration-test-trigger
         name: Check integration test trigger
@@ -109,16 +115,26 @@ jobs:
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
     environment: PyPi-deploy
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.12
+
+      # IMPORTANT: always build sdist, and then the wheel from
+      # the sdist (like it is currently done here). This is
+      # because we want to ensure that no extra files get
+      # copied, which can be the case with building in-tree.
+      # The MANIFEST.in file ensures that the sdist doesn't
+      # contain any unnecessary files.
       - name: Install requirements and build wheel
         shell: bash -l {0}
         run: |
           python -m pip install build twine
           python -m build .
+
       - name: Publish package
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.github/workflows/remote_package_index_test.yml

@@ -22,18 +22,20 @@ jobs:
         ]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.11.1
 
-      - uses: pyodide/pyodide-actions/download-pyodide@v1
+      - uses: pyodide/pyodide-actions/download-pyodide@v2
         with:
           version: ${{ matrix.pyodide-version }}
           to: dist
 
-      - uses: pyodide/pyodide-actions/install-browser@v1
+      - uses: pyodide/pyodide-actions/install-browser@v2
         with:
           runner: ${{ matrix.test-config.runner }}
           browser: ${{ matrix.test-config.runtime }}

.gitmodules

@@ -0,0 +1,5 @@
+# d8e3b31b734926ebbcaff654279f6855a73e052f, for 24.2 release
+# https://github.com/pypa/packaging/releases/tag/24.2
+[submodule "micropip/_vendored/packaging"]
+	path = micropip/_vendored/packaging
+	url = https://github.com/pypa/packaging/

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 default_language_version:
   python: "3.12"
 
-exclude: (^micropip/externals|^tests/vendored|^tests/test_data)
+exclude: (^micropip/_vendored|^tests/vendored|^tests/test_data)
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: "v5.0.0"
@@ -18,25 +18,24 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.6.9"
+    rev: "v0.8.6"
     hooks:
       - id: ruff
         args: [--fix]
 
   - repo: https://github.com/psf/black
-    rev: "24.8.0"
+    rev: "24.10.0"
     hooks:
       - id: black
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.11.2"
+    rev: "v1.14.1"
     hooks:
       - id: mypy
         args: []
         additional_dependencies: &mypy-deps
           - pytest
 
-
   - repo: https://github.com/codespell-project/codespell
     rev: "v2.3.0"
     hooks:

.readthedocs.yml

@@ -12,6 +12,6 @@ python:
       path: .
 
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.12"

CHANGELOG.md

@@ -6,6 +6,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Fixed
+
+- Fix a bug that prevented non-standard relative urls to be treated as such
+  (the ones that starts with `../` or `./`)
+  [#174](https://github.com/pyodide/micropip/pull/174)
+
+- Fixed an error when calling `micropip.install` with `deps=False` is set.
+  [#187](https://github.com/pyodide/micropip/pull/187)
+
+### Added
+
+- `micropip` now vendors `pypa/packaging` for better reliability.
+  [#178](https://github.com/pyodide/micropip/pull/178)
+- `micropip.install` adds optional `constraints`, similar to `pip install -c`,
+  which refine the version or direct URLs of requested packages and their
+  dependencies. This includes built-in packages, which are now installed after
+  any requested or constrained external packages.
+  [#181](https://github.com/pyodide/micropip/pull/181)
+- `micropip.set_constraints` sets default constraints for later
+  calls to `micropip.install` that do not specify constraints.
+  [#181](https://github.com/pyodide/micropip/pull/181)
+
+## [0.8.0] - 2024/12/15
+
+### Added
+
+- Added support for PEP-658.
+  [#139](https://github.com/pyodide/micropip/pull/139)
+
 ## [0.7.2] - 2024/11/26
 
 ### Fixed

MANIFEST.in

@@ -0,0 +1,16 @@
+exclude micropip/_vendored/packaging/.github/*
+exclude micropip/_vendored/packaging/docs/*
+exclude micropip/_vendored/packaging/tasks/*
+exclude micropip/_vendored/packaging/tests/*
+exclude micropip/_vendored/packaging/.pre-commit-config.yaml
+exclude micropip/_vendored/packaging/.readthedocs.yml
+exclude micropip/_vendored/packaging/CHANGELOG.rst
+exclude micropip/_vendored/packaging/CONTRIBUTING.rst
+exclude micropip/_vendored/packaging/noxfile.py
+exclude micropip/_vendored/packaging/pyproject.toml
+
+include micropip/_vendored/packaging/LICENSE
+include micropip/_vendored/packaging/LICENSE.APACHE
+include micropip/_vendored/packaging/LICENSE.BSD
+
+include micropip/_vendored/packaging/README.rst

docs/project/usage.md

@@ -71,18 +71,79 @@ You can pass multiple packages to `micropip.install`:
 await micropip.install(["pkg1", "pkg2"])
 ```
 
-You can specify additional constraints:
+A dependency can be refined as per the [PEP-508] spec:
+
+[pep-508]: https://peps.python.org/pep-0508
 
 ```python
 await micropip.install("snowballstemmer==2.2.0")
 await micropip.install("snowballstemmer>=2.2.0")
+await micropip.install("snowballstemmer @ https://.../snowballstemmer.*.whl")
 await micropip.install("snowballstemmer[all]")
 ```
 
+### Disabling dependency resolution
+
 micropip does dependency resolution by default, but you can disable it,
 this is useful if you want to install a package that has a dependency
 which is not a pure Python package, but it is not mandatory for your use case:
 
 ```python
 await micropip.install("pkg", deps=False)
 ```
+
+### Constraining indirect dependencies
+
+Dependency resolution can be further customized with optional `constraints`:
+these modify both _direct_ and _indirect_ dependency resolutions, while direct URLs
+in either a requirement or constraint will bypass any other specifiers.
+
+As described in the [`pip` documentation][pip-constraints], each constraint:
+
+[pip-constraints]: https://pip.pypa.io/en/stable/user_guide/#constraints-files
+
+  - _must_ provide a name
+  - _must_ provide exactly one of
+    - a set of version specifiers
+    - a URL
+  - _must not_  request any `[extras]`
+
+Multiple constraints of the same canonical name are merged.
+
+Invalid constraints will be silently discarded, or logged if `verbose` is provided.
+
+```python
+await micropip.install(
+    "pkg",
+    constraints=[
+        "other-pkg==0.1.1",
+        "some-other-pkg<2",
+        "some-other-pkg<3",                           # merged with the above
+        "yet-another-pkg@https://example.com/yet_another_pkg-0.1.2-py3-none-any.whl",
+        # silently discarded                          # why?
+        "yet-another-pkg >=1",                        # previously defined by URL
+        "yet_another_pkg-0.1.2-py3-none-any.whl",     # missing name
+        "something-completely[different] ==0.1.1",    # extras
+        "package-with-no-version",                    # missing version or URL
+        "other-pkg ==0.0.1 ; python_version < '3'",   # not applicable
+    ]
+)
+```
+
+Over-constrained requirements will fail to resolve, leaving the environment unmodified.
+
+```python
+await micropip.install("pkg ==1", constraints=["pkg ==2"])
+# ValueError: Can't find a pure Python 3 wheel for 'pkg==1,==2'.
+```
+
+### Setting default constraints
+
+`micropip.set_constraints` replaces any default constraints for all subsequent
+calls to `micropip.install` that don't specify `constraints`:
+
+```python
+micropip.set_constraints(["other-pkg ==0.1.1"])
+await micropip.install("pkg")                         # uses defaults, if needed
+await micropip.install("another-pkg", constraints=[]) # ignores defaults
+```

micropip/__init__.py

@@ -12,6 +12,7 @@
 install = _package_manager_singleton.install
 set_index_urls = _package_manager_singleton.set_index_urls
 list = _package_manager_singleton.list_packages
+set_constraints = _package_manager_singleton.set_constraints
 freeze = _package_manager_singleton.freeze
 add_mock_package = _package_manager_singleton.add_mock_package
 list_mock_packages = _package_manager_singleton.list_mock_packages

micropip/_compat/_compat_not_in_pyodide.py

@@ -14,6 +14,7 @@
 class CompatibilityNotInPyodide(CompatibilityLayer):
 
     # Vendored from packaging
+    # TODO: use packaging APIs here instead?
     _canonicalize_regex = re.compile(r"[-_.]+")
 
     class HttpStatusError(Exception):