e2e: basic nested in docker/podman testing

Basic testing of minimal container build and execution nested under privileged docker, and rootless podman. The docker/podman containerized singularity is built from the source tree in which the e2e-tests are running. We verify basic nested exec of a container with/without userns (-u), an OCI-Mode container (always uses a userns), and simple build to native SIF. Manual testing performed on Fedora 41, with: Docker version 28.0.0, build f9ced58 podman version 5.3.2
sylabs · Feb 24, 2025 · eab5f64 · eab5f64
1 parent 2d0c131
commit eab5f64
Show file tree

Hide file tree

Showing 3 changed files with 196 additions and 1 deletion.
diff --git a/e2e/nested/nested.go b/e2e/nested/nested.go
@@ -0,0 +1,155 @@
+// Copyright (c) 2025, Sylabs Inc. All rights reserved.
+// This software is licensed under a 3-clause BSD license. Please consult the
+// LICENSE.md file distributed with the sources of this project regarding your
+// rights to use or distribute this software.
+
+package nested
+
+import (
+	"os/exec"
+	"testing"
+
+	"github.com/sylabs/singularity/v4/e2e/internal/e2e"
+	"github.com/sylabs/singularity/v4/e2e/internal/testhelper"
+	"github.com/sylabs/singularity/v4/internal/pkg/test/tool/require"
+)
+
+type ctx struct {
+	env e2e.TestEnv
+}
+
+//nolint:dupl
+func (c ctx) docker(t *testing.T) {
+	require.Command(t, "docker")
+	e2e.EnsureORASImage(t, c.env)
+	e2e.EnsureRegistryOCISIF(t, c.env)
+
+	// Temporary homedir for docker commands, so invoking docker doesn't create
+	// a ~/.docker that may interfere elsewhere.
+	tmpHome, cleanupHome := e2e.MakeTempDir(t, c.env.TestDir, "nested-docker-", "")
+	t.Cleanup(func() { e2e.Privileged(cleanupHome)(t) })
+
+	dockerFile := "testdata/Dockerfile.nested"
+	dockerRef := "singularity-e2e-docker-nested"
+	dockerBuild(t, dockerFile, dockerRef, "../", tmpHome)
+	defer dockerRMI(t, dockerRef, tmpHome)
+
+	// Execution using privileged Docker. Root in outer Docker container.
+	dockerRunPrivileged(t, "version", dockerRef, tmpHome)
+	dockerRunPrivileged(t, "exec", dockerRef, tmpHome, "exec", c.env.OrasTestImage, "/bin/true")
+	dockerRunPrivileged(t, "execUserNS", dockerRef, tmpHome, "exec", "-u", c.env.OrasTestImage, "/bin/true")
+	dockerRunPrivileged(t, "execOCI", dockerRef, tmpHome, "exec", "--oci", c.env.TestRegistryOCISIF, "/bin/true")
+	dockerRunPrivileged(t, "buildSIF", dockerRef, tmpHome, "build", "test.sif", "singularity/examples/library/Singularity")
+}
+
+func dockerBuild(t *testing.T, dockerFile, dockerRef, contextPath, homeDir string) {
+	t.Run("build/"+dockerRef, e2e.Privileged(func(t *testing.T) {
+		cmd := exec.Command("docker", "build", "-t", dockerRef, "-f", dockerFile, contextPath)
+		cmd.Env = append(cmd.Env, "HOME="+homeDir)
+		out, err := cmd.CombinedOutput()
+		t.Log(cmd.Args)
+		if err != nil {
+			t.Fatalf("Failed building docker container.\n%s: %s", err, string(out))
+		}
+	}))
+}
+
+func dockerRMI(t *testing.T, dockerRef, homeDir string) {
+	t.Run("rmi/"+dockerRef, e2e.Privileged(func(t *testing.T) {
+		cmd := exec.Command("docker", "rmi", dockerRef)
+		cmd.Env = append(cmd.Env, "HOME="+homeDir)
+		out, err := cmd.CombinedOutput()
+		t.Log(cmd.Args)
+		if err != nil {
+			t.Fatalf("Failed removing docker container.\n%s: %s", err, string(out))
+		}
+	}))
+}
+
+func dockerRunPrivileged(t *testing.T, name, dockerRef, homeDir string, args ...string) { //nolint:unparam
+	t.Run(name, e2e.Privileged(func(t *testing.T) {
+		cmdArgs := []string{"run", "-i", "--rm", "--privileged", "--network=host", dockerRef}
+		cmdArgs = append(cmdArgs, args...)
+		cmd := exec.Command("docker", cmdArgs...)
+		cmd.Env = append(cmd.Env, "HOME="+homeDir)
+		out, err := cmd.CombinedOutput()
+		t.Log(cmd.Args)
+		if err != nil {
+			t.Fatalf("Failed running docker container.\n%s: %s", err, string(out))
+		}
+	}))
+}
+
+//nolint:dupl
+func (c ctx) podman(t *testing.T) {
+	require.Command(t, "podman")
+	e2e.EnsureORASImage(t, c.env)
+	e2e.EnsureRegistryOCISIF(t, c.env)
+
+	// Temporary homedir for docker commands, so invoking docker doesn't create
+	// a ~/.docker that may interfere elsewhere.
+	tmpHome, cleanupHome := e2e.MakeTempDir(t, c.env.TestDir, "nested-docker-", "")
+	t.Cleanup(func() { e2e.Privileged(cleanupHome)(t) })
+
+	dockerFile := "testdata/Dockerfile.nested"
+	dockerRef := "localhost/singularity-e2e-podman-nested"
+	podmanBuild(t, dockerFile, dockerRef, "../", tmpHome)
+	defer podmanRMI(t, dockerRef, tmpHome)
+
+	// Rootless podman - fake userns root in outer podman container.
+	podmanRun(t, "version", dockerRef, tmpHome)
+	podmanRun(t, "exec", dockerRef, tmpHome, "exec", c.env.OrasTestImage, "/bin/true")
+	podmanRun(t, "execUserNS", dockerRef, tmpHome, "exec", "-u", c.env.OrasTestImage, "/bin/true")
+	podmanRun(t, "execOCI", dockerRef, tmpHome, "exec", "--oci", c.env.TestRegistryOCISIF, "/bin/true")
+	podmanRun(t, "buildSIF", dockerRef, tmpHome, "build", "test.sif", "singularity/examples/library/Singularity")
+}
+
+func podmanBuild(t *testing.T, dockerFile, dockerRef, contextPath, homeDir string) {
+	t.Run("build/"+dockerRef, func(t *testing.T) {
+		cmd := exec.Command("podman", "build", "-t", dockerRef, "-f", dockerFile, contextPath)
+		cmd.Env = append(cmd.Env, "HOME="+homeDir)
+		out, err := cmd.CombinedOutput()
+		t.Log(cmd.Args)
+		if err != nil {
+			t.Fatalf("Failed building podman container.\n%s: %s", err, string(out))
+		}
+	})
+}
+
+func podmanRMI(t *testing.T, dockerRef, homeDir string) {
+	t.Run("rmi/"+dockerRef, func(t *testing.T) {
+		cmd := exec.Command("podman", "rmi", dockerRef)
+		cmd.Env = append(cmd.Env, "HOME="+homeDir)
+		out, err := cmd.CombinedOutput()
+		t.Log(cmd.Args)
+		if err != nil {
+			t.Fatalf("Failed removing podman container.\n%s: %s", err, string(out))
+		}
+	})
+}
+
+func podmanRun(t *testing.T, name, dockerRef, homeDir string, args ...string) { //nolint:unparam
+	t.Run(name, func(t *testing.T) {
+		cmdArgs := []string{"run", "-i", "--rm", "--privileged", "--network=host", dockerRef}
+		cmdArgs = append(cmdArgs, args...)
+		cmd := exec.Command("podman", cmdArgs...)
+		cmd.Env = append(cmd.Env, "HOME="+homeDir)
+		out, err := cmd.CombinedOutput()
+		t.Log(cmd.Args)
+		if err != nil {
+			t.Fatalf("Failed running podman container.\n%s: %s", err, string(out))
+		}
+	})
+}
+
+// E2ETests is the main func to trigger the test suite
+func E2ETests(env e2e.TestEnv) testhelper.Tests {
+	c := ctx{
+		env: env,
+	}
+
+	return testhelper.Tests{
+		"Docker": c.docker,
+		"Porman": c.podman,
+	}
+}
diff --git a/e2e/suite.go b/e2e/suite.go
@@ -1,5 +1,5 @@
 // Copyright (c) 2020, Control Command Inc. All rights reserved.
-// Copyright (c) 2019-2022 Sylabs Inc. All rights reserved.
+// Copyright (c) 2019-2025 Sylabs Inc. All rights reserved.
 // Copyright (c) Contributors to the Apptainer project, established as
 //   Apptainer a Series of LF Projects LLC.
 // This software is licensed under a 3-clause BSD license. Please consult the
@@ -40,6 +40,7 @@ import (
 	"github.com/sylabs/singularity/v4/e2e/instance"
 	"github.com/sylabs/singularity/v4/e2e/key"
 	"github.com/sylabs/singularity/v4/e2e/keyserver"
+	"github.com/sylabs/singularity/v4/e2e/nested"
 	"github.com/sylabs/singularity/v4/e2e/oci"
 	"github.com/sylabs/singularity/v4/e2e/overlay"
 	"github.com/sylabs/singularity/v4/e2e/plugin"
@@ -85,6 +86,7 @@ var e2eGroups = map[string]testhelper.Group{
 	"INSTANCE":       instance.E2ETests,
 	"KEY":            key.E2ETests,
 	"KEYSERVER":      keyserver.E2ETests,
+	"NESTED":         nested.E2ETests,
 	"OCI":            oci.E2ETests,
 	"OVERLAY":        overlay.E2ETests,
 	"PLUGIN":         plugin.E2ETests,

diff --git a/e2e/testdata/Dockerfile.nested b/e2e/testdata/Dockerfile.nested
@@ -0,0 +1,38 @@
+FROM ubuntu:24.10
+
+ENV GOLANG_VERSION=1.24.0
+ENV ARCH=amd64
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install system packages
+RUN apt-get update && apt-get install -y autoconf \
+    automake \
+    crun \
+    cryptsetup \
+    fuse2fs \
+    fuse \
+    fuse-overlayfs \
+    git \
+    libfuse-dev \
+    libglib2.0-dev \
+    libseccomp-dev \
+    libsubid-dev \
+    libtool \
+    make \
+    pkg-config \
+    squashfs-tools \
+    squashfs-tools-ng \
+    tzdata \
+    uidmap \
+    wget
+
+# Install GO
+RUN wget -O /tmp/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz https://go.dev/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz && tar -C /usr/local -xzf /tmp/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz
+
+# Install SingularityCE
+ADD . singularity
+RUN cd singularity && git clean -fdx && ./mconfig && make -C builddir && make -C builddir install
+
+ENTRYPOINT ["/usr/local/bin/singularity"]
+CMD ["version"]