From ea0dfba31bafa4b64c7a5f2c7a42a5c941bb65f8 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 18 Sep 2024 19:44:41 -0700 Subject: [PATCH] Prepare the 2.20.0 release. (#2539) Also turn on CI IT sharding for a ~40% wall time reduction, fixing up some existing test issues this smoked out. --- .github/workflows/ci.yml | 94 +++++++++++--------- CHANGES.md | 25 +++++- pex/version.py | 2 +- tests/integration/resolve/test_issue_1907.py | 14 +-- tests/integration/test_integration.py | 19 ++-- tests/integration/test_issue_1179.py | 6 +- tests/integration/test_issue_940.py | 9 +- tests/test_resolver.py | 9 +- 8 files changed, 113 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eedcfa89d..ee4bb41fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,10 +6,6 @@ defaults: env: _PEX_TEST_DEV_ROOT: ${{ github.workspace }}/.pex_dev _PEX_TEST_POS_ARGS: "--color --devpi --devpi-timeout 15.0 --shutdown-devpi -vvs" - # We use this to skip exposing same-versioned Pythons present on Linux hosts. These otherwise can - # collide when attempting to load libpython..so and lead to mysterious errors - # importing builtins like `fcntl` as outlined in https://github.com/pex-tool/pex/issues/1391. - _PEX_TEST_PYENV_VERSIONS: "2.7 3.7 3.10" _PEX_PEXPECT_TIMEOUT: 10 # We have integration tests that exercise `--scie` support and these can trigger downloads from # GitHub Releases that needed elevated rate limit quota, which this gives. @@ -64,36 +60,51 @@ jobs: # N.B.: The name of this job key (linux-tests) is depended on by scrips/build_cache_image.py. In # particular, the tox-env matrix list is used to ensure the cache covers all Linux CI jobs. linux-tests: - name: ./dtox.sh -e ${{ matrix.tox-env }} + name: ./dtox.sh -e ${{ matrix.tox-env }} ${{ matrix.pex-test-pos-args }} needs: org-check runs-on: ubuntu-22.04 strategy: matrix: - tox-env: - - py27-pip20 - # N.B.: Although we support Python 3.5 with a published universal wheel, Pex does not - # build under Python 3.5 or 3.6; so we skip to Python 3.7 for tests which must be able to - # build a Pex distribution both as part of the tox setup and in many test cases as well. - - py37-pip20 - - py311-pip20 - - py311-pip22_3_1 - - py311-pip23_1_2 - - py312-pip24_2 - - py313-pip24_2 - - pypy310-pip20 - - pypy310-pip22_3_1 - - pypy310-pip23_1_2 - - py27-pip20-integration - - py37-pip22_3_1-integration - - py37-pip23_1_2-integration - - py311-pip20-integration - - py311-pip22_3_1-integration - - py311-pip23_1_2-integration - - py312-pip24_2-integration - - py313-pip24_2-integration - - pypy310-pip20-integration - - pypy310-pip22_3_1-integration - - pypy310-pip23_1_2-integration + include: + # Unit tests: + # ----------- + - tox-env: py27-pip20 + - tox-env: py37-pip22_3_1 + - tox-env: py311-pip23_3_2 + - tox-env: py312-pip24_2 + - tox-env: py313-pip24_2 + - tox-env: pypy310-pip24_2 + + # Integration tests, split most into two shards: + # ---------------------------------------------- + + # CPython 2.7 is fast enough not to require sharding. + - tox-env: py27-pip20-integration + + - tox-env: py37-pip22_3_1-integration + pex-test-pos-args: --shard 1/2 + - tox-env: py37-pip22_3_1-integration + pex-test-pos-args: --shard 2/2 + + - tox-env: py311-pip23_3_2-integration + pex-test-pos-args: --shard 1/2 + - tox-env: py311-pip23_3_2-integration + pex-test-pos-args: --shard 2/2 + + - tox-env: py312-pip24_2-integration + pex-test-pos-args: --shard 1/2 + - tox-env: py312-pip24_2-integration + pex-test-pos-args: --shard 2/2 + + - tox-env: py313-pip24_2-integration + pex-test-pos-args: --shard 1/2 + - tox-env: py313-pip24_2-integration + pex-test-pos-args: --shard 2/2 + + - tox-env: pypy310-pip24_2-integration + pex-test-pos-args: --shard 1/2 + - tox-env: pypy310-pip24_2-integration + pex-test-pos-args: --shard 2/2 steps: - name: Free Up Disk Space uses: jlumbroso/free-disk-space@v1.3.1 @@ -125,9 +136,10 @@ jobs: # This is needed to get pexpect tests working under PyPy running under docker. export TERM="xterm" BASE_MODE=pull CACHE_MODE=pull \ - ./dtox.sh -e ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }} + ./dtox.sh -e ${{ matrix.tox-env }} -- \ + ${{ env._PEX_TEST_POS_ARGS }} ${{ matrix.pex-test-pos-args }} mac-tests: - name: tox -e ${{ matrix.tox-env }} + name: tox -e ${{ matrix.tox-env }} ${{ matrix.pex-test-pos-args }} needs: org-check runs-on: macos-12 strategy: @@ -139,15 +151,12 @@ jobs: - python-version: [ 3, 12 ] tox-env: py312-pip24_2-integration tox-env-python: python3.11 + pex-test-pos-args: --shard 1/2 + - python-version: [ 3, 12 ] + tox-env: py312-pip24_2-integration + tox-env-python: python3.11 + pex-test-pos-args: --shard 2/2 steps: - - name: Calculate Pythons to Expose - id: calculate-pythons-to-expose - run: | - skip="" - if [[ "$(uname -s)" == "Linux" ]]; then - skip="${{ env._PEX_TEST_PYENV_VERSIONS }}" - fi - echo "skip=${skip}" >> $GITHUB_OUTPUT - name: Checkout Pex uses: actions/checkout@v4 with: @@ -160,8 +169,6 @@ jobs: python-version: "${{ join(matrix.python-version, '.') }}" - name: Expose Pythons uses: pantsbuild/actions/expose-pythons@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb - with: - skip: "${{ steps.calculate-pythons-to-expose.outputs.skip }}" - name: Restore Cached Pyenv Interpreters id: restore-pyenv-interpreters uses: actions/cache/restore@v4 @@ -190,7 +197,8 @@ jobs: with: path: repo/tox.ini python: ${{ matrix.tox-env-python }} - tox-env: ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }} + tox-env: >- + ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }} ${{ matrix.pex-test-pos-args }} - name: Cache Pyenv Interpreters uses: actions/cache/save@v4 if: github.ref == 'refs/heads/main' diff --git a/CHANGES.md b/CHANGES.md index 1a5653d02..9183f6c1c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,28 @@ # Release Notes +## 2.20.0 + +This release adds the `--pip-log` alias for the existing +`--preserve-pip-download-log` option as well as the ability to specify +the log file path. So, to debug a resolve, you can now specify +`--pip-log log.txt` and Pex will deposit the Pip resolve log to +`log.txt` in the current directory for easy tailing or post-resolve +inspection. In addition, the log file itself is more useful in some +cases. When you specify any abbreviated `--platform` targets, those +targets calculated wheel compatibility tags are included in the Pip +log. Also, when multiple targets are specified, their log outputs are +now merged at the end of the resolve in a serialized fashion with +prefixes on each log line indicating which target the log line +corresponds to. + +In addition, a race in Pex's PEP-517 implementation that could (rarely) +lead to spurious metadata generation errors or sdist creation errors is +fixed. + +* Fix intermittent PEP-517 failures. (#2540) +* Plumb `--pip-version` to Platform tag calculation. (#2538) +* Add the ability to specify the `--pip-log` path. (#2536) + ## 2.19.1 This release fixes a regression introduced by #2512 in the 2.19.0 @@ -8,7 +31,7 @@ Instead of failing certain builds that used to succeed, Pex now warns that the resulting PEX may fail at runtime and that `--complete-platform` should be used instead. -# Only warn when `--platform` resolves fail tag checks. (#2533) +* Only warn when `--platform` resolves fail tag checks. (#2533) ## 2.19.0 diff --git a/pex/version.py b/pex/version.py index a241b0c71..9c268901c 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.19.1" +__version__ = "2.20.0" diff --git a/tests/integration/resolve/test_issue_1907.py b/tests/integration/resolve/test_issue_1907.py index 2f569c12d..dea95f003 100644 --- a/tests/integration/resolve/test_issue_1907.py +++ b/tests/integration/resolve/test_issue_1907.py @@ -18,7 +18,7 @@ from pex.pex import PEX from pex.pip.version import PipVersion from pex.typing import TYPE_CHECKING -from testing import PY_VER, data, run_pex_command +from testing import IS_PYPY, PY_VER, data, run_pex_command from testing.cli import run_pex3 if TYPE_CHECKING: @@ -168,11 +168,15 @@ def test_pre_resolved_dists_offline( offline = os.path.join(str(tmpdir), "offline") os.mkdir(offline) - # In order to go offline and still be able to build sdists, we need both the un-vendored Pip and - # its basic build requirements. - if PipVersion.DEFAULT is not PipVersion.VENDORED: + if IS_PYPY or PipVersion.DEFAULT is not PipVersion.VENDORED: args = [sys.executable, "-m", "pip", "wheel", "-w", offline] - args.extend(str(req) for req in PipVersion.DEFAULT.requirements) + if PipVersion.DEFAULT is not PipVersion.VENDORED: + # In order to go offline and still be able to build sdists, we need both the un-vendored Pip and + # its basic build requirements. + args.extend(str(req) for req in PipVersion.DEFAULT.requirements) + if IS_PYPY: + # For PyPy, we need extra build dependencies for argon2-cffi-bindings. + args.append("setuptools_scm") subprocess.check_call(args) for dist in glob.glob(os.path.join(dists, "*")): diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 77fd75d09..b095a16a2 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -1979,10 +1979,15 @@ def test_require_hashes(tmpdir): run_pex_command(args=["-r", requirements, "-o", requests_pex]).assert_success() subprocess.check_call(args=[requests_pex, "-c", "import requests"]) - # The hash checking mode should also work in constraints context. - run_pex_command( - args=["--constraints", requirements, "requests", "-o", requests_pex] - ).assert_success() + result = run_pex_command(args=["--constraints", requirements, "requests", "-o", requests_pex]) + # The hash checking mode should also work in constraints context for Pip prior to 23.2 when + # Pip got more strict about the contents of constraints files (just specifiers and markers; no + # extras, hashes, etc.). + if PipVersion.DEFAULT < PipVersion.v23_2: + result.assert_success() + else: + result.assert_failure() + subprocess.check_call(args=[requests_pex, "-c", "import requests"]) with open(requirements, "w") as fp: @@ -2007,13 +2012,9 @@ def test_require_hashes(tmpdir): as_requirements_result = run_pex_command(args=["-r", requirements]) as_requirements_result.assert_failure() - # The hash checking mode should also work in constraints context. - as_constraints_result = run_pex_command(args=["--constraints", requirements, "requests"]) - as_constraints_result.assert_failure() - error_lines = { re.sub(r"\s+", " ", line.strip()): index - for index, line in enumerate(as_constraints_result.error.splitlines()) + for index, line in enumerate(as_requirements_result.error.splitlines()) } index = error_lines["pip: Expected sha512 worse"] assert ( diff --git a/tests/integration/test_issue_1179.py b/tests/integration/test_issue_1179.py index 66488f71c..20733b031 100644 --- a/tests/integration/test_issue_1179.py +++ b/tests/integration/test_issue_1179.py @@ -6,16 +6,18 @@ import pytest +from pex.pip.version import PipVersion from pex.targets import LocalInterpreter from testing import run_pex_command @pytest.mark.skipif( - sys.version_info[:2] >= (3, 12), + sys.version_info[:2] >= (3, 12) or PipVersion.DEFAULT >= PipVersion.v23_2, reason=( "There is an indirect urllib3 dependency which embeds six which uses a meta path importer " "that only implements the PEP-302 finder spec and not the modern spec. Only the modern " - "finder spec is supported by Python 3.12+." + "finder spec is supported by Python 3.12+. Also, Pip 23.2 dropped support for the legacy " + "resolver, which this test needs." ), ) def test_pip_2020_resolver_engaged(): diff --git a/tests/integration/test_issue_940.py b/tests/integration/test_issue_940.py index 5e1453888..e6bbad6dc 100644 --- a/tests/integration/test_issue_940.py +++ b/tests/integration/test_issue_940.py @@ -7,6 +7,7 @@ import pytest +from pex.pip.version import PipVersion from pex.typing import TYPE_CHECKING from testing import built_wheel, run_pex_command, run_simple_pex @@ -15,8 +16,12 @@ @pytest.mark.skipif( - sys.version_info[:2] >= (3, 12), - reason="We need to use setuptools<66 but Python 3.12+ require greater.", + sys.version_info[:2] >= (3, 12) or PipVersion.DEFAULT >= PipVersion.v24_1, + reason=( + "We need to use setuptools<66 but Python 3.12+ require greater. We also need to avoid " + "Pip>=24.1 which upgrades its vendored packaging to a version that rejects invalid " + "versions" + ), ) def test_resolve_arbitrary_equality(tmpdir): # type: (Any) -> None diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 7757a820d..8cd19a298 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -17,6 +17,7 @@ from pex.common import safe_copy, safe_mkdtemp, temporary_dir from pex.dist_metadata import Distribution, Requirement from pex.interpreter import PythonInterpreter +from pex.pip.version import PipVersion from pex.resolve import abbreviated_platforms from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion @@ -536,8 +537,12 @@ def assert_dist( @pytest.mark.skipif( - sys.version_info[:2] >= (3, 12), - reason="We need to use setuptools<66 ut Python 3.12+ require greater.", + sys.version_info[:2] >= (3, 12) or PipVersion.DEFAULT >= PipVersion.v24_1, + reason=( + "We need to use setuptools<66, but Python 3.12+ require greater. We also need to avoid " + "Pip>=24.1 which upgrades its vendored packaging to a version that rejects invalid " + "versions" + ), ) def test_resolve_arbitrary_equality_issues_940(): # type: (...) -> None