Set up matrix testing for PDM versions. (#4)

.buildkite/pipeline.yml

@@ -1,9 +1,10 @@
 plugin_base: &plugin_base
     service-account-name: monorepo-ci
-    image: gcr.io/embark-shared/ml/ci-runner@sha256:db9dc3fd8e333c1a990b6465b2c62822afdd4859c1a7f59621157fea05d5a81f
+    image: gcr.io/embark-shared/ml/ci-runner@sha256:34407f69dae312c51ee1b30df13e2d39d2fda624017c76a42cd9ecbce127ddae
     default-secret-name: buildkite-k8s-plugin
     always-pull: false
     use-agent-node-affinity: true
+    mount-hostpath: /tmp/pdm-cache:/root/.cache/pdm
 
 agents: &agent
   cluster: builds-fi-2
@@ -61,8 +62,11 @@ steps:
         command: bash .buildkite/run-bandit.sh
         << : *tiny
 
-      - label: ":pytest: Run tests"
-        command: bash .buildkite/run-pytest.sh
+      - label: ":pytest: Run tests @ {{matrix}}"
+        matrix:
+          - "pdm"
+          - "pdm24"
+        command: bash .buildkite/run-pytest.sh {{matrix}}
         << : *small
 
   - wait

.buildkite/run-pytest.sh

@@ -5,10 +5,10 @@ source .buildkite/install-repo.sh
 echo --- Running pytest
 
 EXIT_CODE=0
-pdm run pytest --color=yes tests > errors.txt || EXIT_CODE=$?
+($1 run pytest --color=yes tests --pdm-bin $1 | tee errors.txt) || EXIT_CODE=$?
 
 if [ $EXIT_CODE -ne 0 ]; then
-	cat << EOF | buildkite-agent annotate --style "error" --context "pytest"
+	cat << EOF | buildkite-agent annotate --style "error" --context "pytest-$1"
 :warning: Tests failed. Please see below errors and correct any issues. You can run tests locally with \`pdm run pytest tests pdm-plugin-torch\`.
 
 \`\`\`term
@@ -17,7 +17,7 @@ $(cat errors.txt)
 
 EOF
 else
-	buildkite-agent annotate "✅ All tests passed." --style "success" --context "pytest"
+	buildkite-agent annotate "✅ All tests passed for $1." --style "success" --context "pytest-$1"
 fi
 
 exit $EXIT_CODE

pdm-plugin-torch/pdm_plugin_torch/main.py

@@ -282,7 +282,7 @@ def get_settings(project: Project):
         return project.pyproject["tool"]["pdm"]["plugins"]["torch"]
 
     else:
-        return project.tool_settings["plugins"]["torch"]
+        return project.pyproject.settings["plugins"]["torch"]
 
 
 class TorchCommand(BaseCommand):

pyproject.toml

@@ -8,13 +8,16 @@ authors = [
 requires-python = ">=3.8"
 readme = "README.md"
 license = {text = "MIT"}
+dependencies = [ ]
 
 [tool.pdm.dev-dependencies]
 tools = [
    "pytest~=6.0",
    "black~=22.1",
    "bandit~=1.7",
    "isort~=5.10",
+   "pytest",
+   "pytest-xdist",
 ]
 
 docs = [
@@ -36,8 +39,8 @@ build-backend = "pdm.pep517.api"
 pdm_plugin_torch = "pdm_plugin_torch.main:torch_plugin"
 
 [tool.pdm.build]
-package-dir = "src"
-includes = ["src/pdm_plugin_torch"]
+package-dir = "pdm-plugin-torch"
+includes = ["pdm-plugin-torch"]
 source-includes = ["tests", "CHANGELOG.md", "LICENSE", "README.md"]
 # editables backend doesn't work well with namespace packages
 editable-backend = "path"

tests/__init__.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+
+
+FIXTURES = Path(__file__).parent / "fixtures"

tests/conftest.py

@@ -0,0 +1,64 @@
+import os
+
+from typing import Dict, Tuple
+
+import pytest
+
+
+os.environ.update(CI="1", PDM_CHECK_UPDATE="0")
+
+
+# store history of failures per test class name and per index in parametrize (if parametrize used)
+_test_failed_incremental: Dict[str, Dict[Tuple[int, ...], str]] = {}
+
+
+def pytest_runtest_makereport(item, call):
+    if "incremental" in item.keywords:
+        # incremental marker is used
+        if call.excinfo is not None:
+            # the test has failed
+            # retrieve the class name of the test
+            cls_name = str(item.cls)
+            # retrieve the index of the test if parametrize is used in combination with incremental
+            parametrize_index = (
+                tuple(item.callspec.indices.values())
+                if hasattr(item, "callspec")
+                else ()
+            )
+            # retrieve the name of the test function
+            test_name = item.originalname or item.name
+            # store in _test_failed_incremental the original name of the failed test
+            _test_failed_incremental.setdefault(cls_name, {}).setdefault(
+                parametrize_index, test_name
+            )
+
+
+def pytest_runtest_setup(item):
+    if "incremental" in item.keywords:
+        # retrieve the class name of the test
+        cls_name = str(item.cls)
+        # check if a previous test has failed for this class
+        if cls_name in _test_failed_incremental:
+            # retrieve the index of the test if parametrize is used in combination with incremental
+            parametrize_index = (
+                tuple(item.callspec.indices.values())
+                if hasattr(item, "callspec")
+                else ()
+            )
+            # retrieve the name of the first test function to fail for this class name and index
+            test_name = _test_failed_incremental[cls_name].get(parametrize_index, None)
+            # if name found, test has failed for the combination of class name & test name
+            if test_name is not None:
+                pytest.xfail("previous test failed ({})".format(test_name))
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "incremental: mark tests in class to run in series with early out"
+    )
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--pdm-bin", action="store", default="pdm", help="the pdm binary to run"
+    )

tests/fixtures/cpu-only/pyproject.toml

@@ -0,0 +1,31 @@
+[project]
+name = "test-cpu-only"
+authors = [
+    {name = "Tom Solberg", email = "[email protected]"},
+]
+requires-python = "~=3.8.0"
+license = {text = "MIT"}
+dependencies = []
+description = ""
+version = "0.0.01"
+
+[build-system]
+requires = ["pdm-pep517"]
+build-backend = "pdm.pep517.api"
+
+[tool.pdm.plugins.torch]
+dependencies = [
+   "torch==1.10.2"
+]
+lockfile = "torch.lock"
+enable-cpu = true
+
+enable-rocm = false
+rocm-versions = ["4.2"]
+
+enable-cuda = false
+cuda-versions = ["cu111", "cu113"]
+
+[tool.pdm.scripts]
+post_install = "pdm plugin add ../../"
+post_lock = "pdm torch lock"

tests/test_lock.py

@@ -0,0 +1,80 @@
+import os
+import shutil
+import subprocess
+
+from pathlib import Path
+from unittest import mock
+
+import pytest
+
+from tests import FIXTURES
+
+
+PLUGIN_DIR = os.path.abspath(f"{__file__}/../..")
+
+
+def copytree(src: Path, dst: Path) -> None:
+    if not dst.exists():
+        dst.mkdir(parents=True)
+    for subpath in src.iterdir():
+        if subpath.is_dir():
+            copytree(subpath, dst / subpath.name)
+        else:
+            shutil.copy2(subpath, dst)
+
+
+def make_entry_point(plugin):
+    ret = mock.Mock()
+    ret.load.return_value = plugin
+    return ret
+
+
+def tmpdir_project(project_name, dest):
+    source = FIXTURES / project_name
+    copytree(source, dest)
+
+
+@pytest.fixture
+def pdm(request):
+    pdm_name = request.config.getoption("--pdm-bin")
+
+    def _invoker(args, dir):
+        output = subprocess.check_output([pdm_name, *args], cwd=dir)
+        return output
+
+    return _invoker
+
+
+class TestPdmVariants:
+    @staticmethod
+    def test_install_plugin(tmpdir, pdm):
+        output = pdm(["self", "add", PLUGIN_DIR], tmpdir)
+        assert output == b"Installation succeeds.\n"
+
+    @staticmethod
+    def test_lock_check_fails(tmpdir, pdm):
+        import subprocess
+
+        tmpdir_project("cpu-only", tmpdir)
+        with pytest.raises(subprocess.CalledProcessError):
+            pdm(["torch", "lock", "--check"], tmpdir)
+
+    @staticmethod
+    def test_lock_plugin_check_succeeds(tmpdir, pdm):
+        tmpdir_project("cpu-only", tmpdir)
+        pdm(["torch", "lock"], tmpdir)
+        pdm(["torch", "lock", "--check"], tmpdir)
+
+    @staticmethod
+    def test_install_fails(tmpdir, pdm):
+        import subprocess
+
+        tmpdir_project("cpu-only", tmpdir)
+        with pytest.raises(subprocess.CalledProcessError):
+            pdm(["torch", "install", "cpu"], tmpdir)
+
+    @staticmethod
+    def test_install_succeeds(tmpdir, pdm):
+        tmpdir_project("cpu-only", tmpdir)
+        pdm(["torch", "lock"], tmpdir)
+        pdm(["torch", "install", "cpu"], tmpdir)