Implement long-running tests using the actual user environment

Usage: `make test-full` or `make coverage-full`

Hardware tests are currently disabled!

Makefile

@@ -54,19 +54,31 @@ lint/black: ## check style with black
 
 lint: lint/flake8 lint/black ## check style
 
-test: ## run tests quickly with the default Python
-	python setup.py test
+test: ## run tests quickly
+	pytest tests
+
+test-full: ## also run more complex tests
+	# pytest --run-slow --run-user-context --run-hardware tests
+	pytest --run-slow --run-user-context tests
 
 test-all: ## run tests on every Python version with tox
 	tox
 
 coverage: ## check code coverage quickly with the default Python
 	# coverage run --source mlonmcu setup.py test
+	# coverage run --source mlonmcu -m pytest tests
 	coverage run --source mlonmcu -m pytest tests
 	coverage report -m
 	coverage html
 	$(BROWSER) htmlcov/index.html
 
+coverage-full: ## check code coverage quickly with the default Python
+	# coverage run --source mlonmcu -m pytest --run-slow --run-user-context --run-hardware tests
+	coverage run --source mlonmcu -m pytest --run-slow --run-user-context tests
+	coverage report -m
+	coverage html
+	$(BROWSER) htmlcov/index.html
+
 docs: ## generate Sphinx HTML documentation, including API docs
 	rm -f docs/mlonmcu.rst
 	rm -f docs/modules.rst

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+markers =
+    hardware: mark a test as a hardware test.
+    user_context: mark test to use user actual user context.
+    slow: mark test as slow.

tests/conftest.py

@@ -16,7 +16,43 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import pytest
+
 pytest_plugins = [
-   "tests.fixtures",
+    "tests.fixtures",
 ]
 
+
+def pytest_addoption(parser):
+    parser.addoption("--run-slow", action="store_true", default=False, help="run slow tests")
+    parser.addoption("--run-user-context", action="store_true", default=False, help="run tests using user context")
+    parser.addoption("--run-hardware", action="store_true", default=False, help="run tests using real hardware")
+
+
+# def pytest_configure(config):
+#     config.addinivalue_line("markers", "slow: mark test as slow to run")
+
+
+def pytest_collection_modifyitems(config, items):
+    slow = False
+    if config.getoption("--run-slow"):
+        # --run-slow given in cli: do not skip slow tests
+        slow = True
+    user_context = False
+    if config.getoption("--run-user-context"):
+        # --run-context given in cli: do not skip user context tests
+        user_context = True
+    hardware = False
+    if config.getoption("--run-hardware"):
+        # --run-hardware given in cli: do not skip hardware tests
+        hardware = True
+    skip_slow = pytest.mark.skip(reason="need --run-slow option to run")
+    skip_user_context = pytest.mark.skip(reason="need --run-context option to run")
+    skip_hardware = pytest.mark.skip(reason="need --run-hardware option to run")
+    for item in items:
+        if not slow and "slow" in item.keywords:
+            item.add_marker(skip_slow)
+        if not user_context and "user_context" in item.keywords:
+            item.add_marker(skip_user_context)
+        if not hardware and "hardware" in item.keywords:
+            item.add_marker(skip_hardware)

tests/fixtures.py

@@ -19,7 +19,14 @@
 import os
 import pytest
 import mock
+import tempfile
 import urllib.request
+from pathlib import Path
+from io import BytesIO
+from zipfile import ZipFile
+
+import mlonmcu  # TODO: fix this bad?
+
 
 @pytest.fixture()
 def fake_config_home(tmp_path):
@@ -47,27 +54,47 @@ def fake_working_directory(tmp_path, monkeypatch):
     monkeypatch.chdir(str(cwd))
     yield cwd
 
-import mlonmcu  # TODO: fix this bad?
 
 @pytest.fixture()
 def fake_context():
-    class FakeTaskCache():
+    class FakeTaskCache:
         def __init__(self):
             self._vars = {}
-    class FakeEnvironment():
+
+    class FakeEnvironment:
         def __init__(self):
             self.paths = {}  # TODO: get rid of PathConfig if possible?
-    class FakeContext():
+
+    class FakeContext:
         def __init__(self):
             self.environment = FakeEnvironment()
             self.cache = FakeTaskCache()
+
     context = FakeContext()
     yield context
 
+
 @pytest.fixture()
 def example_elf_file(request, tmp_path):
     name = request.param
     elf_path = tmp_path / name
     url = f"https://github.com/JonathanSalwan/binary-samples/raw/master/{name}"
     urllib.request.urlretrieve(url, elf_path)
     yield str(elf_path)
+
+
+@pytest.fixture()
+def user_context():
+    with mlonmcu.context.MlonMcuContext() as context:
+        yield context
+
+
+@pytest.fixture(scope="session")
+def models_dir():
+    with tempfile.TemporaryDirectory() as tmp_path:
+        url = "https://codeload.github.com/tum-ei-eda/mlonmcu-models/zip/refs/heads/main"
+        resp = urllib.request.urlopen(url)
+        with ZipFile(BytesIO(resp.read())) as zip_file:
+            zip_file.extractall(tmp_path)
+        models_path = Path(tmp_path) / "mlonmcu-models-main"
+        yield models_path