diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4e455a3..f038955 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,9 +1,10 @@ plugin_base: &plugin_base service-account-name: monorepo-ci - image: gcr.io/embark-shared/ml/ci-runner@sha256:db9dc3fd8e333c1a990b6465b2c62822afdd4859c1a7f59621157fea05d5a81f + image: gcr.io/embark-shared/ml/ci-runner@sha256:34407f69dae312c51ee1b30df13e2d39d2fda624017c76a42cd9ecbce127ddae default-secret-name: buildkite-k8s-plugin always-pull: false use-agent-node-affinity: true + mount-hostpath: /tmp/pdm-cache:/root/.cache/pdm agents: &agent cluster: builds-fi-2 @@ -61,8 +62,11 @@ steps: command: bash .buildkite/run-bandit.sh << : *tiny - - label: ":pytest: Run tests" - command: bash .buildkite/run-pytest.sh + - label: ":pytest: Run tests @ {{matrix}}" + matrix: + - "pdm" + - "pdm24" + command: bash .buildkite/run-pytest.sh {{matrix}} << : *small - wait diff --git a/.buildkite/run-pytest.sh b/.buildkite/run-pytest.sh index d9cb20c..9179297 100644 --- a/.buildkite/run-pytest.sh +++ b/.buildkite/run-pytest.sh @@ -5,10 +5,10 @@ source .buildkite/install-repo.sh echo --- Running pytest EXIT_CODE=0 -pdm run pytest --color=yes tests > errors.txt || EXIT_CODE=$? +($1 run pytest --color=yes tests --pdm-bin $1 | tee errors.txt) || EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then - cat << EOF | buildkite-agent annotate --style "error" --context "pytest" + cat << EOF | buildkite-agent annotate --style "error" --context "pytest-$1" :warning: Tests failed. Please see below errors and correct any issues. You can run tests locally with \`pdm run pytest tests pdm-plugin-torch\`. \`\`\`term @@ -17,7 +17,7 @@ $(cat errors.txt) EOF else - buildkite-agent annotate "✅ All tests passed." --style "success" --context "pytest" + buildkite-agent annotate "✅ All tests passed for $1." --style "success" --context "pytest-$1" fi exit $EXIT_CODE diff --git a/pdm-plugin-torch/pdm_plugin_torch/main.py b/pdm-plugin-torch/pdm_plugin_torch/main.py index ac39af1..7603666 100644 --- a/pdm-plugin-torch/pdm_plugin_torch/main.py +++ b/pdm-plugin-torch/pdm_plugin_torch/main.py @@ -282,7 +282,7 @@ def get_settings(project: Project): return project.pyproject["tool"]["pdm"]["plugins"]["torch"] else: - return project.tool_settings["plugins"]["torch"] + return project.pyproject.settings["plugins"]["torch"] class TorchCommand(BaseCommand): diff --git a/pdm.lock b/pdm.lock index c897116..46d11b7 100644 --- a/pdm.lock +++ b/pdm.lock @@ -83,6 +83,12 @@ version = "0.17.1" requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" summary = "Docutils -- Python Documentation Utilities" +[[package]] +name = "execnet" +version = "1.9.0" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +summary = "execnet: rapid multi-Python deployment" + [[package]] name = "gitdb" version = "4.0.10" @@ -252,6 +258,16 @@ dependencies = [ "toml", ] +[[package]] +name = "pytest-xdist" +version = "3.2.0" +requires_python = ">=3.7" +summary = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +dependencies = [ + "execnet>=1.1", + "pytest>=6.2.0", +] + [[package]] name = "pytz" version = "2022.7.1" @@ -416,7 +432,7 @@ summary = "Backport of pathlib-compatible object wrapper for zip files" [metadata] lock_version = "4.1" -content_hash = "sha256:a85f2821af3f18e972f19319b741efee751ce38214099d8abe728a6cf2132aaf" +content_hash = "sha256:d69f7223a004c512b359e659a061c6f51529fc9347ef33b91b58ffda635e5813" [metadata.files] "alabaster 0.7.13" = [ @@ -558,6 +574,10 @@ content_hash = "sha256:a85f2821af3f18e972f19319b741efee751ce38214099d8abe728a6cf {url = "https://files.pythonhosted.org/packages/4c/17/559b4d020f4b46e0287a2eddf2d8ebf76318fd3bd495f1625414b052fdc9/docutils-0.17.1.tar.gz", hash = "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125"}, {url = "https://files.pythonhosted.org/packages/4c/5e/6003a0d1f37725ec2ebd4046b657abb9372202655f96e76795dca8c0063c/docutils-0.17.1-py2.py3-none-any.whl", hash = "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61"}, ] +"execnet 1.9.0" = [ + {url = "https://files.pythonhosted.org/packages/7a/3c/b5ac9fc61e1e559ced3e40bf5b518a4142536b34eb274aa50dff29cb89f5/execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, + {url = "https://files.pythonhosted.org/packages/81/c0/3072ecc23f4c5e0a1af35e3a222855cfd9c80a1a105ca67be3b6172637dd/execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, +] "gitdb 4.0.10" = [ {url = "https://files.pythonhosted.org/packages/21/a6/35f83efec687615c711fe0a09b67e58f6d1254db27b1013119de46f450bd/gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, {url = "https://files.pythonhosted.org/packages/4b/47/dc98f3d5d48aa815770e31490893b92c5f1cd6c6cf28dd3a8ae0efffac14/gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, @@ -694,6 +714,10 @@ content_hash = "sha256:a85f2821af3f18e972f19319b741efee751ce38214099d8abe728a6cf {url = "https://files.pythonhosted.org/packages/40/76/86f886e750b81a4357b6ed606b2bcf0ce6d6c27ad3c09ebf63ed674fc86e/pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, {url = "https://files.pythonhosted.org/packages/4b/24/7d1f2d2537de114bdf1e6875115113ca80091520948d370c964b88070af2/pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] +"pytest-xdist 3.2.0" = [ + {url = "https://files.pythonhosted.org/packages/e0/a0/bf843762eb7ac86d793903b2c42215d6d2f1d3c0f4265da29e8c2b006b44/pytest-xdist-3.2.0.tar.gz", hash = "sha256:fa10f95a2564cd91652f2d132725183c3b590d9fdcdec09d3677386ecf4c1ce9"}, + {url = "https://files.pythonhosted.org/packages/e5/62/78212674a3fab3e15c81812a3889480ed5ac4c82b1ab9c37c74834f30920/pytest_xdist-3.2.0-py3-none-any.whl", hash = "sha256:336098e3bbd8193276867cc87db8b22903c3927665dff9d1ac8684c02f597b68"}, +] "pytz 2022.7.1" = [ {url = "https://files.pythonhosted.org/packages/03/3e/dc5c793b62c60d0ca0b7e58f1fdd84d5aaa9f8df23e7589b39cc9ce20a03/pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, {url = "https://files.pythonhosted.org/packages/2e/09/fbd3c46dce130958ee8e0090f910f1fe39e502cc5ba0aadca1e8a2b932e5/pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, diff --git a/pyproject.toml b/pyproject.toml index 2ff1882..b5e5eec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ authors = [ requires-python = ">=3.8" readme = "README.md" license = {text = "MIT"} +dependencies = [ ] [tool.pdm.dev-dependencies] tools = [ @@ -15,6 +16,8 @@ tools = [ "black~=22.1", "bandit~=1.7", "isort~=5.10", + "pytest", + "pytest-xdist", ] docs = [ @@ -36,8 +39,8 @@ build-backend = "pdm.pep517.api" pdm_plugin_torch = "pdm_plugin_torch.main:torch_plugin" [tool.pdm.build] -package-dir = "src" -includes = ["src/pdm_plugin_torch"] +package-dir = "pdm-plugin-torch" +includes = ["pdm-plugin-torch"] source-includes = ["tests", "CHANGELOG.md", "LICENSE", "README.md"] # editables backend doesn't work well with namespace packages editable-backend = "path" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..5fc0a58 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,4 @@ +from pathlib import Path + + +FIXTURES = Path(__file__).parent / "fixtures" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8337d0f --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,64 @@ +import os + +from typing import Dict, Tuple + +import pytest + + +os.environ.update(CI="1", PDM_CHECK_UPDATE="0") + + +# store history of failures per test class name and per index in parametrize (if parametrize used) +_test_failed_incremental: Dict[str, Dict[Tuple[int, ...], str]] = {} + + +def pytest_runtest_makereport(item, call): + if "incremental" in item.keywords: + # incremental marker is used + if call.excinfo is not None: + # the test has failed + # retrieve the class name of the test + cls_name = str(item.cls) + # retrieve the index of the test if parametrize is used in combination with incremental + parametrize_index = ( + tuple(item.callspec.indices.values()) + if hasattr(item, "callspec") + else () + ) + # retrieve the name of the test function + test_name = item.originalname or item.name + # store in _test_failed_incremental the original name of the failed test + _test_failed_incremental.setdefault(cls_name, {}).setdefault( + parametrize_index, test_name + ) + + +def pytest_runtest_setup(item): + if "incremental" in item.keywords: + # retrieve the class name of the test + cls_name = str(item.cls) + # check if a previous test has failed for this class + if cls_name in _test_failed_incremental: + # retrieve the index of the test if parametrize is used in combination with incremental + parametrize_index = ( + tuple(item.callspec.indices.values()) + if hasattr(item, "callspec") + else () + ) + # retrieve the name of the first test function to fail for this class name and index + test_name = _test_failed_incremental[cls_name].get(parametrize_index, None) + # if name found, test has failed for the combination of class name & test name + if test_name is not None: + pytest.xfail("previous test failed ({})".format(test_name)) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "incremental: mark tests in class to run in series with early out" + ) + + +def pytest_addoption(parser): + parser.addoption( + "--pdm-bin", action="store", default="pdm", help="the pdm binary to run" + ) diff --git a/tests/fixtures/cpu-only/pyproject.toml b/tests/fixtures/cpu-only/pyproject.toml new file mode 100644 index 0000000..c40e2a4 --- /dev/null +++ b/tests/fixtures/cpu-only/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "test-cpu-only" +authors = [ + {name = "Tom Solberg", email = "me@sbg.dev"}, +] +requires-python = "~=3.8.0" +license = {text = "MIT"} +dependencies = [] +description = "" +version = "0.0.01" + +[build-system] +requires = ["pdm-pep517"] +build-backend = "pdm.pep517.api" + +[tool.pdm.plugins.torch] +dependencies = [ + "torch==1.10.2" +] +lockfile = "torch.lock" +enable-cpu = true + +enable-rocm = false +rocm-versions = ["4.2"] + +enable-cuda = false +cuda-versions = ["cu111", "cu113"] + +[tool.pdm.scripts] +post_install = "pdm plugin add ../../" +post_lock = "pdm torch lock" diff --git a/tests/test_dummy.py b/tests/test_dummy.py deleted file mode 100644 index e2ac78e..0000000 --- a/tests/test_dummy.py +++ /dev/null @@ -1,7 +0,0 @@ -""" - -""" - - -def test_nothing(): - pass diff --git a/tests/test_lock.py b/tests/test_lock.py new file mode 100644 index 0000000..eca990b --- /dev/null +++ b/tests/test_lock.py @@ -0,0 +1,80 @@ +import os +import shutil +import subprocess + +from pathlib import Path +from unittest import mock + +import pytest + +from tests import FIXTURES + + +PLUGIN_DIR = os.path.abspath(f"{__file__}/../..") + + +def copytree(src: Path, dst: Path) -> None: + if not dst.exists(): + dst.mkdir(parents=True) + for subpath in src.iterdir(): + if subpath.is_dir(): + copytree(subpath, dst / subpath.name) + else: + shutil.copy2(subpath, dst) + + +def make_entry_point(plugin): + ret = mock.Mock() + ret.load.return_value = plugin + return ret + + +def tmpdir_project(project_name, dest): + source = FIXTURES / project_name + copytree(source, dest) + + +@pytest.fixture +def pdm(request): + pdm_name = request.config.getoption("--pdm-bin") + + def _invoker(args, dir): + output = subprocess.check_output([pdm_name, *args], cwd=dir) + return output + + return _invoker + + +class TestPdmVariants: + @staticmethod + def test_install_plugin(tmpdir, pdm): + output = pdm(["self", "add", PLUGIN_DIR], tmpdir) + assert output == b"Installation succeeds.\n" + + @staticmethod + def test_lock_check_fails(tmpdir, pdm): + import subprocess + + tmpdir_project("cpu-only", tmpdir) + with pytest.raises(subprocess.CalledProcessError): + pdm(["torch", "lock", "--check"], tmpdir) + + @staticmethod + def test_lock_plugin_check_succeeds(tmpdir, pdm): + tmpdir_project("cpu-only", tmpdir) + pdm(["torch", "lock"], tmpdir) + pdm(["torch", "lock", "--check"], tmpdir) + + @staticmethod + def test_install_fails(tmpdir, pdm): + import subprocess + + tmpdir_project("cpu-only", tmpdir) + with pytest.raises(subprocess.CalledProcessError): + pdm(["torch", "install", "cpu"], tmpdir) + + @staticmethod + def test_install_succeeds(tmpdir, pdm): + tmpdir_project("cpu-only", tmpdir) + pdm(["torch", "lock"], tmpdir) + pdm(["torch", "install", "cpu"], tmpdir)