Compose Deploy support for Docker CDI (#36)

* feat: Compose deploy support for CDI devices * add GPU section to README
aksiksi · Aug 31, 2024 · 9145716 · 9145716
1 parent 3f3f499
commit 9145716
Show file tree

Hide file tree

Showing 25 changed files with 221 additions and 39 deletions.
diff --git a/README.md b/README.md
@@ -177,6 +177,51 @@ You can do one of the following:
 sudo systemctl start podman-compose-myservice-root.target
 ```
 
+### Nvidia GPU Support
+
+1. Enable CDI support in your NixOS config:
+
+```nix
+{
+  hardware.nvidia-container-toolkit.enable = true;
+}
+```
+
+**Docker only:**
+
+Make sure you are running Docker 25+:
+
+```nix
+{
+  virtualisation.docker.package = pkgs.docker_25;
+}
+```
+
+2. Pass in CDI devices via either `devices` **or** `deploy` (both map to the same thing under the hood):
+
+```yaml
+services:
+  myservice:
+    # ... other fields
+
+    # Option 1
+    devices:
+      - nvidia.com/gpu=all
+    # Option 2
+    deploy:
+      resources:
+        reservations:
+          devices:
+            # Driver must be set to "cdi" - all others are ignored.
+            - driver: cdi
+              device_ids:
+                - nvidia.com/gpu=all
+
+    # Required for Podman.
+    security_opt:
+      - label=disable
+```
+
 ### NixOS Version Support Policy
 
 I always aim to support the **latest** stable version of NixOS (24.05 at the

diff --git a/compose.go b/compose.go
@@ -571,6 +571,16 @@ func (g *Generator) buildNixContainer(service types.ServiceConfig, networkMap ma
 			if reservations.NanoCPUs != 0 {
 				c.ExtraOptions = append(c.ExtraOptions, "--cpus="+strconv.FormatFloat(float64(reservations.NanoCPUs), 'f', -1, 32))
 			}
+
+			// CDI GPU support.
+			for _, device := range reservations.Devices {
+				if strings.ToLower(device.Driver) != "cdi" {
+					continue
+				}
+				for _, deviceID := range device.IDs {
+					c.ExtraOptions = append(c.ExtraOptions, "--device="+deviceID)
+				}
+			}
 		}
 	}
 

diff --git a/nix_test.go b/nix_test.go
@@ -274,3 +274,12 @@ func TestEmptyEnv(t *testing.T) {
 	}
 	runSubtestsWithGenerator(t, g)
 }
+
+func TestDeployDevices(t *testing.T) {
+	composePath, _ := getPaths(t, false)
+	g := &Generator{
+		Inputs:  []string{composePath},
+		Project: NewProject("test"),
+	}
+	runSubtestsWithGenerator(t, g)
+}
diff --git a/testdata/TestBasic.docker.nix b/testdata/TestBasic.docker.nix
@@ -286,8 +286,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."docker-traefik" = {

diff --git a/testdata/TestBasic.podman.nix b/testdata/TestBasic.podman.nix
@@ -284,8 +284,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."podman-traefik" = {

diff --git a/testdata/TestBasicAutoFormat.docker.nix b/testdata/TestBasicAutoFormat.docker.nix
@@ -258,8 +258,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."docker-traefik" = {

diff --git a/testdata/TestBasicAutoFormat.podman.nix b/testdata/TestBasicAutoFormat.podman.nix
@@ -256,8 +256,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."podman-traefik" = {

diff --git a/testdata/TestDeployDevices.compose.yml b/testdata/TestDeployDevices.compose.yml
@@ -0,0 +1,19 @@
+services:
+  test:
+    image: nginx:latest
+    security_opt:
+      - label=disable
+    devices:
+      - nvidia.com/gpu=abc
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: cdi
+              device_ids:
+                - nvidia.com/gpu=all
+            - driver: ignore-me
+              device_ids:
+                - unknown
+    restart: unless-stopped
+
diff --git a/testdata/TestDeployDevices.docker.nix b/testdata/TestDeployDevices.docker.nix
@@ -0,0 +1,68 @@
+{ pkgs, lib, ... }:
+
+{
+  # Runtime
+  virtualisation.docker = {
+    enable = true;
+    autoPrune.enable = true;
+  };
+  virtualisation.oci-containers.backend = "docker";
+
+  # Containers
+  virtualisation.oci-containers.containers."test-test" = {
+    image = "nginx:latest";
+    log-driver = "journald";
+    autoStart = false;
+    extraOptions = [
+      "--device=nvidia.com/gpu=abc"
+      "--device=nvidia.com/gpu=all"
+      "--network-alias=test"
+      "--network=test_default"
+      "--security-opt=label=disable"
+    ];
+  };
+  systemd.services."docker-test-test" = {
+    serviceConfig = {
+      Restart = lib.mkOverride 500 "always";
+      RestartMaxDelaySec = lib.mkOverride 500 "1m";
+      RestartSec = lib.mkOverride 500 "100ms";
+      RestartSteps = lib.mkOverride 500 9;
+    };
+    after = [
+      "docker-network-test_default.service"
+    ];
+    requires = [
+      "docker-network-test_default.service"
+    ];
+    partOf = [
+      "docker-compose-test-root.target"
+    ];
+    wantedBy = [
+      "docker-compose-test-root.target"
+    ];
+  };
+
+  # Networks
+  systemd.services."docker-network-test_default" = {
+    path = [ pkgs.docker ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      ExecStop = "docker network rm -f test_default";
+    };
+    script = ''
+      docker network inspect test_default || docker network create test_default
+    '';
+    partOf = [ "docker-compose-test-root.target" ];
+    wantedBy = [ "docker-compose-test-root.target" ];
+  };
+
+  # Root service
+  # When started, this will automatically create all resources and start
+  # the containers. When stopped, this will teardown all resources.
+  systemd.targets."docker-compose-test-root" = {
+    unitConfig = {
+      Description = "Root target generated by compose2nix.";
+    };
+  };
+}
diff --git a/testdata/TestDeployDevices.podman.nix b/testdata/TestDeployDevices.podman.nix
@@ -0,0 +1,70 @@
+{ pkgs, lib, ... }:
+
+{
+  # Runtime
+  virtualisation.podman = {
+    enable = true;
+    autoPrune.enable = true;
+    dockerCompat = true;
+    defaultNetwork.settings = {
+      # Required for container networking to be able to use names.
+      dns_enabled = true;
+    };
+  };
+  virtualisation.oci-containers.backend = "podman";
+
+  # Containers
+  virtualisation.oci-containers.containers."test-test" = {
+    image = "nginx:latest";
+    log-driver = "journald";
+    autoStart = false;
+    extraOptions = [
+      "--device=nvidia.com/gpu=abc"
+      "--device=nvidia.com/gpu=all"
+      "--network-alias=test"
+      "--network=test_default"
+      "--security-opt=label=disable"
+    ];
+  };
+  systemd.services."podman-test-test" = {
+    serviceConfig = {
+      Restart = lib.mkOverride 500 "always";
+    };
+    after = [
+      "podman-network-test_default.service"
+    ];
+    requires = [
+      "podman-network-test_default.service"
+    ];
+    partOf = [
+      "podman-compose-test-root.target"
+    ];
+    wantedBy = [
+      "podman-compose-test-root.target"
+    ];
+  };
+
+  # Networks
+  systemd.services."podman-network-test_default" = {
+    path = [ pkgs.podman ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      ExecStop = "podman network rm -f test_default";
+    };
+    script = ''
+      podman network inspect test_default || podman network create test_default
+    '';
+    partOf = [ "podman-compose-test-root.target" ];
+    wantedBy = [ "podman-compose-test-root.target" ];
+  };
+
+  # Root service
+  # When started, this will automatically create all resources and start
+  # the containers. When stopped, this will teardown all resources.
+  systemd.targets."podman-compose-test-root" = {
+    unitConfig = {
+      Description = "Root target generated by compose2nix.";
+    };
+  };
+}
diff --git a/testdata/TestEnvFilesOnly.docker.nix b/testdata/TestEnvFilesOnly.docker.nix
@@ -265,8 +265,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."docker-traefik" = {

diff --git a/testdata/TestEnvFilesOnly.podman.nix b/testdata/TestEnvFilesOnly.podman.nix
@@ -263,8 +263,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."podman-traefik" = {

diff --git a/testdata/TestNoWriteNixSetup.docker.nix b/testdata/TestNoWriteNixSetup.docker.nix
@@ -285,8 +285,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."docker-traefik" = {

diff --git a/testdata/TestNoWriteNixSetup.podman.nix b/testdata/TestNoWriteNixSetup.podman.nix
@@ -278,8 +278,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."podman-traefik" = {

diff --git a/testdata/TestOverrideSystemdStopTimeout.docker.nix b/testdata/TestOverrideSystemdStopTimeout.docker.nix
@@ -295,8 +295,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."docker-traefik" = {

diff --git a/testdata/TestOverrideSystemdStopTimeout.podman.nix b/testdata/TestOverrideSystemdStopTimeout.podman.nix
@@ -293,8 +293,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."podman-traefik" = {

diff --git a/testdata/TestProject.docker.nix b/testdata/TestProject.docker.nix
@@ -291,8 +291,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."docker-traefik" = {

diff --git a/testdata/TestProject.podman.nix b/testdata/TestProject.podman.nix
@@ -289,8 +289,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."podman-traefik" = {

diff --git a/testdata/TestRemoveVolumes.docker.nix b/testdata/TestRemoveVolumes.docker.nix
@@ -291,8 +291,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."docker-traefik" = {

diff --git a/testdata/TestRemoveVolumes.podman.nix b/testdata/TestRemoveVolumes.podman.nix
@@ -289,8 +289,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."podman-traefik" = {

diff --git a/testdata/TestSystemdMount.docker.nix b/testdata/TestSystemdMount.docker.nix
@@ -305,8 +305,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."docker-traefik" = {

diff --git a/testdata/TestSystemdMount.podman.nix b/testdata/TestSystemdMount.podman.nix
@@ -303,8 +303,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."podman-traefik" = {

diff --git a/testdata/TestUpheldBy.docker.nix b/testdata/TestUpheldBy.docker.nix
@@ -297,8 +297,6 @@
       "--log-opt=max-file=3"
       "--log-opt=max-size=10m"
       "--network=container:sabnzbd"
-      "--runtime=nvidia"
-      "--security-opt=label=disable"
     ];
   };
   systemd.services."docker-traefik" = {