Merge branch 'main' into remove-artifacts

.pre-commit-config.yaml

@@ -17,10 +17,12 @@ repos:
           - id: yamlfmt
             args: [--preserve-quotes]
 
-    - repo: https://github.com/psf/black
-      rev: 24.3.0
+    - repo: https://github.com/astral-sh/ruff-pre-commit
+      rev: v0.3.5
       hooks:
-          - id: black
+          - id: ruff-format
+          - id: ruff
+            args: [--fix, --exit-non-zero-on-fix, --show-fixes]
 
     - repo: https://github.com/python-jsonschema/check-jsonschema
       rev: "0.27.3"

docker-bake.hcl

@@ -84,6 +84,9 @@ target "base-with-services" {
     "PGSQL_VERSION" = "${PGSQL_VERSION}"
   }
 }
+# PYTHON_MINOR_VERSION is a Python version string
+# without the patch version (e.g. "3.9")
+# Used to construct paths to Python site-packages folder.
 target "lab" {
   inherits = ["lab-meta"]
   context = "stack/lab"
@@ -94,6 +97,7 @@ target "lab" {
   args = {
     "AIIDALAB_VERSION"      = "${AIIDALAB_VERSION}"
     "AIIDALAB_HOME_VERSION" = "${AIIDALAB_HOME_VERSION}"
+    "PYTHON_MINOR_VERSION" = join(".", slice(split(".", "${PYTHON_VERSION}"), 0, 2))
   }
 }
 target "full-stack" {

dodo.py

@@ -2,9 +2,8 @@
 import platform
 from pathlib import Path
 
-from doit.tools import title_with_actions
-
 import docker
+from doit.tools import title_with_actions
 from dunamai import Version
 
 _DOCKER_CLIENT = docker.from_env()
@@ -74,13 +73,13 @@ def generate_version_override(
 
         Path("docker-bake.override.json").write_text(
             json.dumps(
-                dict(
-                    VERSION=version,
-                    REGISTRY=registry,
-                    TARGETS=targets,
-                    ORGANIZATION=organization,
-                    PLATFORMS=platforms,
-                )
+                {
+                    "VERSION": version,
+                    "REGISTRY": registry,
+                    "TARGETS": targets,
+                    "ORGANIZATION": organization,
+                    "PLATFORMS": platforms,
+                }
             )
         )
 

ruff.toml

@@ -0,0 +1,25 @@
+line-length = 88
+show-fixes = true
+target-version = "py38"
+
+[lint]
+ignore = ["E501", "E402", "B904", "TRY003"]
+select = [
+  "A",    # flake8-builtins
+  "ARG",  # flake8-unused-arguments
+  "B",    # flake8-bugbear
+  "C4",   # flake8-comprehensions
+  "E",    # pycodestyle
+  "F",    # pyflakes
+  "I",    # isort
+  "N",    # pep8-naming
+  "PLE",  # pylint error rules
+  "PLW",  # pylint warning rules
+  "PLC",  # pylint convention rules
+  "RUF",  # ruff-specific rules
+  "TRY",  # Tryceratops
+  "UP"    # pyupgrade
+]
+
+[lint.per-file-ignores]
+"tests/*" = ["ARG001", "B017"]

stack/base/Dockerfile

@@ -20,8 +20,7 @@ ARG AIIDA_VERSION
 # We pin aiida-core to the exact installed version,
 # to prevent accidental upgrade or downgrade, that might
 # induce DB migration or break shared dependencies of AiiDAlab Apps.
-RUN echo "pip==23.3.1" > /opt/requirements.txt && \
-  echo "aiida-core==${AIIDA_VERSION}" >> /opt/requirements.txt
+RUN echo "aiida-core==${AIIDA_VERSION}" > /opt/requirements.txt
 
 # Install the shared requirements.
 RUN mamba install --yes \
@@ -32,11 +31,15 @@ RUN mamba install --yes \
      fix-permissions "${CONDA_DIR}" && \
      fix-permissions "/home/${NB_USER}"
 
-# Pin shared requirements in the base environemnt.
+# Pin shared requirements in the conda base environment.
 RUN cat /opt/requirements.txt | xargs -I{} conda config --system --add pinned_packages {}
 
+# Upgrade pip to latest
+RUN pip install --upgrade --no-cache-dir pip
 # Configure pip to use requirements file as constraints file.
-ENV PIP_CONSTRAINT=/opt/requirements.txt
+ENV PIP_CONSTRAINT /opt/requirements.txt
+# Ensure that pip installs packages to ~/.local by default
+ENV PIP_USER 1
 
 # Enable verdi autocompletion.
 RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \

stack/lab/Dockerfile

@@ -41,20 +41,20 @@ ARG AIIDALAB_HOME_VERSION
 RUN git clone https://github.com/aiidalab/aiidalab-home && \
      cd aiidalab-home && \
      git checkout v"${AIIDALAB_HOME_VERSION}" && \
-     pip install --quiet --no-cache-dir "./" && \
+     pip install --no-user --quiet --no-cache-dir "./" && \
      fix-permissions "./" && \
      fix-permissions "${CONDA_DIR}" && \
      fix-permissions "/home/${NB_USER}"
 
-# Install and enable appmode.
-RUN git clone https://github.com/oschuett/appmode.git && \
-     cd appmode && \
-     git checkout v0.8.0
-COPY gears.svg ./appmode/appmode/static/gears.svg
-RUN pip install ./appmode --no-cache-dir && \
-     jupyter nbextension enable --py --sys-prefix appmode && \
+# Install and enable appmode, turning Jupyter notebooks to Apps
+RUN pip install appmode==0.8.0 --no-cache-dir --no-user
+# Enable appmode extension
+RUN jupyter nbextension enable --py --sys-prefix appmode && \
      jupyter serverextension enable --py --sys-prefix appmode
 
+# Swap appmode icon for AiiDAlab gears icon, shown during app load
+COPY --chown=${NB_UID}:${NB_GID} gears.svg ${CONDA_DIR}/share/jupyter/nbextensions/appmode/gears.svg
+
 # Copy start-up scripts for AiiDAlab.
 COPY before-notebook.d/* /usr/local/bin/before-notebook.d/
 
@@ -107,8 +107,5 @@ ENV NOTEBOOK_ARGS \
      "--TerminalManager.cull_interval=60"
 
 # Set up the logo of notebook interface
-COPY --chown=${NB_UID}:${NB_GID} aiidalab-wide-logo.png /tmp/notebook-logo.png
-
-# The directory location of logo.png is in the `${CONDA_DIR}/lib/python3.9/site-packages/notebook/static/base/images/logo.png`,
-# but the python version may change in the future, thus we use the wildcard to match the python version.
-RUN mv /tmp/notebook-logo.png ${CONDA_DIR}/lib/python3*/site-packages/notebook/static/base/images/logo.png
+ARG PYTHON_MINOR_VERSION
+COPY --chown=${NB_UID}:${NB_GID} aiidalab-wide-logo.png ${CONDA_DIR}/lib/python${PYTHON_MINOR_VERSION}/site-packages/notebook/static/base/images/logo.png

tests/conftest.py

@@ -3,9 +3,10 @@
 
 import pytest
 import requests
-
 from requests.exceptions import ConnectionError
 
+TARGETS = ("base", "lab", "base-with-services", "full-stack")
+
 
 def is_responsive(url):
     try:
@@ -16,19 +17,34 @@ def is_responsive(url):
         return False
 
 
+def target_checker(value):
+    msg = f"Invalid image target '{value}', must be one of: {TARGETS}"
+    if value not in TARGETS:
+        raise pytest.UsageError(msg)
+    return value
+
+
 def pytest_addoption(parser):
     parser.addoption(
-        "--variant",
+        "--target",
         action="store",
-        default="base",
-        help="Variant (image name) of the docker-compose file to use.",
+        required=True,
+        help="target (image name) of the docker-compose file to use.",
+        type=target_checker,
     )
 
 
+@pytest.fixture(scope="session")
+def target(pytestconfig):
+    return pytestconfig.getoption("target")
+
+
 @pytest.fixture(scope="session")
 def docker_compose_file(pytestconfig):
-    variant = pytestconfig.getoption("variant")
-    return f"stack/docker-compose.{variant}.yml"
+    target = pytestconfig.getoption("target")
+    compose_file = f"stack/docker-compose.{target}.yml"
+    print(f"Using docker compose file {compose_file}")
+    return compose_file
 
 
 @pytest.fixture(scope="session")
@@ -54,14 +70,30 @@ def execute(command, user=None, **kwargs):
             command = f"exec -T --user={user} aiidalab {command}"
         else:
             command = f"exec -T aiidalab {command}"
-        return docker_compose.execute(command, **kwargs)
+        out = docker_compose.execute(command, **kwargs)
+        return out.decode()
 
     return execute
 
 
 @pytest.fixture
 def nb_user(aiidalab_exec):
-    return aiidalab_exec("bash -c 'echo \"${NB_USER}\"'").decode().strip()
+    return aiidalab_exec("bash -c 'echo \"${NB_USER}\"'").strip()
+
+
+@pytest.fixture
+def pip_install(aiidalab_exec, nb_user):
+    """Temporarily install package via pip"""
+    package = None
+
+    def _pip_install(pkg, **args):
+        nonlocal package
+        package = pkg
+        return aiidalab_exec(f"pip install {pkg}", **args)
+
+    yield _pip_install
+    if package:
+        aiidalab_exec(f"pip uninstall --yes {package}")
 
 
 @pytest.fixture(scope="session")

tests/test-base.py → tests/test_base.py

@@ -1,54 +1,66 @@
 """This module contains tests for the base image, which are AiiDA and package management related tests."""
 
-import pytest
 import json
+
+import pytest
 from packaging.version import parse
 
 
-@pytest.mark.parametrize("incompatible_version", ["2.3.0"])
-def test_prevent_pip_install_of_incompatible_aiida_version(
-    aiidalab_exec, nb_user, aiida_version, incompatible_version
+@pytest.mark.parametrize("pkg_manager", ["pip", "mamba"])
+def test_prevent_installation_of_aiida(
+    aiidalab_exec, nb_user, aiida_version, pkg_manager
 ):
-    package_manager = "pip"
+    """aiida-core is pinned to the exact version in the container,
+    test that both pip and mamba refuse to install a different version"""
+
+    incompatible_version = "2.3.0"
     assert parse(aiida_version) != parse(incompatible_version)
+
     # Expected to succeed, although should be a no-op.
-    aiidalab_exec(
-        f"{package_manager} install aiida-core=={aiida_version}", user=nb_user
-    )
+    aiidalab_exec(f"{pkg_manager} install aiida-core=={aiida_version}", user=nb_user)
     with pytest.raises(Exception):
         aiidalab_exec(
-            f"{package_manager} install aiida-core=={incompatible_version}",
+            f"{pkg_manager} install aiida-core=={incompatible_version}",
             user=nb_user,
         )
 
 
 def test_correct_python_version_installed(aiidalab_exec, python_version):
-    info = json.loads(aiidalab_exec("mamba list --json --full-name python").decode())[0]
+    info = json.loads(aiidalab_exec("mamba list --json --full-name python"))[0]
     assert info["name"] == "python"
     assert parse(info["version"]) == parse(python_version)
 
 
 def test_create_conda_environment(aiidalab_exec, nb_user):
-    output = aiidalab_exec("conda create -y -n tmp", user=nb_user).decode().strip()
+    output = aiidalab_exec("conda create -y -n tmp", user=nb_user).strip()
     assert "conda activate tmp" in output
     # New conda environments should be created in ~/.conda/envs/
-    output = aiidalab_exec("conda env list", user=nb_user).decode().strip()
+    output = aiidalab_exec("conda env list", user=nb_user).strip()
     assert f"/home/{nb_user}/.conda/envs/tmp" in output
 
 
-def test_pip_check(aiidalab_exec):
-    aiidalab_exec("pip check")
-
-
 def test_correct_aiida_version_installed(aiidalab_exec, aiida_version):
-    info = json.loads(
-        aiidalab_exec("mamba list --json --full-name aiida-core").decode()
-    )[0]
+    cmd = "mamba list --json --full-name aiida-core"
+    info = json.loads(aiidalab_exec(cmd))[0]
     assert info["name"] == "aiida-core"
     assert parse(info["version"]) == parse(aiida_version)
 
 
 def test_path_local_pip(aiidalab_exec, nb_user):
     """test that the pip local bin path ~/.local/bin is added to PATH"""
-    output = aiidalab_exec("bash -c 'echo \"${PATH}\"'", user=nb_user).decode()
+    output = aiidalab_exec("bash -c 'echo \"${PATH}\"'", user=nb_user)
     assert f"/home/{nb_user}/.local/bin" in output
+
+
+def test_pip_user_install(aiidalab_exec, pip_install, nb_user):
+    """Test that pip installs packages to ~/.local/ by default"""
+    import email
+
+    # We use 'tuna' as an example of python-only package without dependencies
+    pkg = "tuna"
+    pip_install(pkg)
+    output = aiidalab_exec(f"pip show {pkg}")
+
+    # `pip show` output is in the RFC-compliant email header format
+    msg = email.message_from_string(output)
+    assert msg.get("Location").startswith(f"/home/{nb_user}/.local/")

tests/test-base-with-services.py → tests/test_base_with_services.py

@@ -1,15 +1,25 @@
 """Services related tests."""
 
 import json
+
+import pytest
 from packaging.version import parse
 
+# Tests in this file should pass for the following images
+TESTED_TARGETS = ("base-with-services", "full-stack")
+
+
+@pytest.fixture(autouse=True)
+def skip_if_incompatible_target(target):
+    if target in TESTED_TARGETS:
+        yield
+    else:
+        pytest.skip()
+
 
 def test_correct_pgsql_version_installed(aiidalab_exec, pgsql_version):
-    info = json.loads(
-        aiidalab_exec(
-            "mamba list -n aiida-core-services --json --full-name postgresql"
-        ).decode()
-    )[0]
+    cmd = "mamba list -n aiida-core-services --json --full-name postgresql"
+    info = json.loads(aiidalab_exec(cmd))[0]
     assert info["name"] == "postgresql"
     assert parse(info["version"]).major == parse(pgsql_version).major
 
@@ -18,5 +28,4 @@ def test_rabbitmq_can_start(aiidalab_exec):
     """Test the rabbitmq-server can start, the output should be empty if
     the command is successful."""
     output = aiidalab_exec("mamba run -n aiida-core-services rabbitmq-server -detached")
-
-    assert output == b""
+    assert output == ""

tests/test-common.py → tests/test_common.py

@@ -9,14 +9,18 @@ def test_notebook_service_available(notebook_service):
 
 
 def test_verdi_status(aiidalab_exec, nb_user):
-    output = aiidalab_exec("verdi status", user=nb_user).decode().strip()
+    output = aiidalab_exec("verdi status", user=nb_user).strip()
     assert "Connected to RabbitMQ" in output
     assert "Daemon is running" in output
 
 
 def test_ssh_agent_is_running(aiidalab_exec, nb_user):
-    output = aiidalab_exec("ps aux | grep ssh-agent", user=nb_user).decode().strip()
+    output = aiidalab_exec("ps aux | grep ssh-agent", user=nb_user).strip()
     assert "ssh-agent" in output
 
     # also check only one ssh-agent process is running
     assert len(output.splitlines()) == 1
+
+
+def test_pip_check(aiidalab_exec):
+    aiidalab_exec("pip check")