From 4990f441a7c328a95815ec0d4df98a72d52e1c3f Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 24 Feb 2025 15:40:55 +0000 Subject: [PATCH] e2e: basic nested in docker/podman testing Basic testing of minimal container build and execution nested under privileged docker, and rootless podman. The docker/podman containerized singularity is built from the source tree in which the e2e-tests are running. We verify basic nested exec of a container with/without userns (-u), an OCI-Mode container (always uses a userns), and simple build to native SIF. Manual testing performed on Fedora 41, with: Docker version 28.0.0, build f9ced58 podman version 5.3.2 --- e2e/nested/nested.go | 164 +++++++++++++++++++++++++++++++++ e2e/suite.go | 4 +- e2e/testdata/Dockerfile.nested | 39 ++++++++ 3 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 e2e/nested/nested.go create mode 100644 e2e/testdata/Dockerfile.nested diff --git a/e2e/nested/nested.go b/e2e/nested/nested.go new file mode 100644 index 0000000000..3d7c3fb712 --- /dev/null +++ b/e2e/nested/nested.go @@ -0,0 +1,164 @@ +// Copyright (c) 2025, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package nested + +import ( + "os/exec" + "runtime" + "testing" + + "github.com/sylabs/singularity/v4/e2e/internal/e2e" + "github.com/sylabs/singularity/v4/e2e/internal/testhelper" + "github.com/sylabs/singularity/v4/internal/pkg/test/tool/require" +) + +type ctx struct { + env e2e.TestEnv +} + +//nolint:dupl +func (c ctx) docker(t *testing.T) { + require.Command(t, "docker") + e2e.EnsureORASImage(t, c.env) + e2e.EnsureRegistryOCISIF(t, c.env) + + // Temporary homedir for docker commands, so invoking docker doesn't create + // a ~/.docker that may interfere elsewhere. + tmpHome, cleanupHome := e2e.MakeTempDir(t, c.env.TestDir, "nested-docker-", "") + t.Cleanup(func() { e2e.Privileged(cleanupHome)(t) }) + + dockerFile := "testdata/Dockerfile.nested" + dockerRef := "singularity-e2e-docker-nested" + dockerBuild(t, dockerFile, dockerRef, "../", tmpHome) + defer dockerRMI(t, dockerRef, tmpHome) + + // Execution using privileged Docker. Root in outer Docker container. + dockerRunPrivileged(t, "version", dockerRef, tmpHome) + dockerRunPrivileged(t, "exec", dockerRef, tmpHome, "exec", c.env.OrasTestImage, "/bin/true") + dockerRunPrivileged(t, "execUserNS", dockerRef, tmpHome, "exec", "-u", c.env.OrasTestImage, "/bin/true") + dockerRunPrivileged(t, "execOCI", dockerRef, tmpHome, "exec", "--oci", c.env.TestRegistryOCISIF, "/bin/true") + dockerRunPrivileged(t, "buildSIF", dockerRef, tmpHome, "build", "test.sif", "singularity/examples/library/Singularity") +} + +func dockerBuild(t *testing.T, dockerFile, dockerRef, contextPath, homeDir string) { + t.Run("build/"+dockerRef, e2e.Privileged(func(t *testing.T) { + cmd := exec.Command("docker", "build", + "--build-arg", "GOVERSION="+runtime.Version(), + "--build-arg", "GOOS="+runtime.GOOS, + "--build-arg", "GOARCH="+runtime.GOARCH, + "-t", dockerRef, "-f", dockerFile, contextPath) + cmd.Env = append(cmd.Env, "HOME="+homeDir) + out, err := cmd.CombinedOutput() + t.Log(cmd.Args) + if err != nil { + t.Fatalf("Failed building docker container.\n%s: %s", err, string(out)) + } + })) +} + +func dockerRMI(t *testing.T, dockerRef, homeDir string) { + t.Run("rmi/"+dockerRef, e2e.Privileged(func(t *testing.T) { + cmd := exec.Command("docker", "rmi", dockerRef) + cmd.Env = append(cmd.Env, "HOME="+homeDir) + out, err := cmd.CombinedOutput() + t.Log(cmd.Args) + if err != nil { + t.Fatalf("Failed removing docker container.\n%s: %s", err, string(out)) + } + })) +} + +func dockerRunPrivileged(t *testing.T, name, dockerRef, homeDir string, args ...string) { //nolint:unparam + t.Run(name, e2e.Privileged(func(t *testing.T) { + cmdArgs := []string{"run", "-i", "--rm", "--privileged", "--network=host", dockerRef} + cmdArgs = append(cmdArgs, args...) + cmd := exec.Command("docker", cmdArgs...) + cmd.Env = append(cmd.Env, "HOME="+homeDir) + out, err := cmd.CombinedOutput() + t.Log(cmd.Args) + if err != nil { + t.Fatalf("Failed running docker container.\n%s: %s", err, string(out)) + } + })) +} + +//nolint:dupl +func (c ctx) podman(t *testing.T) { + require.Command(t, "podman") + e2e.EnsureORASImage(t, c.env) + e2e.EnsureRegistryOCISIF(t, c.env) + + // Temporary homedir for docker commands, so invoking docker doesn't create + // a ~/.docker that may interfere elsewhere. + tmpHome, cleanupHome := e2e.MakeTempDir(t, c.env.TestDir, "nested-docker-", "") + t.Cleanup(func() { e2e.Privileged(cleanupHome)(t) }) + + dockerFile := "testdata/Dockerfile.nested" + dockerRef := "localhost/singularity-e2e-podman-nested" + podmanBuild(t, dockerFile, dockerRef, "../", tmpHome) + defer podmanRMI(t, dockerRef, tmpHome) + + // Rootless podman - fake userns root in outer podman container. + podmanRun(t, "version", dockerRef, tmpHome) + podmanRun(t, "exec", dockerRef, tmpHome, "exec", c.env.OrasTestImage, "/bin/true") + podmanRun(t, "execUserNS", dockerRef, tmpHome, "exec", "-u", c.env.OrasTestImage, "/bin/true") + podmanRun(t, "execOCI", dockerRef, tmpHome, "exec", "--oci", c.env.TestRegistryOCISIF, "/bin/true") + podmanRun(t, "buildSIF", dockerRef, tmpHome, "build", "test.sif", "singularity/examples/library/Singularity") +} + +func podmanBuild(t *testing.T, dockerFile, dockerRef, contextPath, homeDir string) { + t.Run("build/"+dockerRef, func(t *testing.T) { + cmd := exec.Command("podman", "build", + "--build-arg", "GOVERSION="+runtime.Version(), + "--build-arg", "GOOS="+runtime.GOOS, + "--build-arg", "GOARCH="+runtime.GOARCH, + "-t", dockerRef, "-f", dockerFile, contextPath) + cmd.Env = append(cmd.Env, "HOME="+homeDir) + out, err := cmd.CombinedOutput() + t.Log(cmd.Args) + if err != nil { + t.Fatalf("Failed building podman container.\n%s: %s", err, string(out)) + } + }) +} + +func podmanRMI(t *testing.T, dockerRef, homeDir string) { + t.Run("rmi/"+dockerRef, func(t *testing.T) { + cmd := exec.Command("podman", "rmi", dockerRef) + cmd.Env = append(cmd.Env, "HOME="+homeDir) + out, err := cmd.CombinedOutput() + t.Log(cmd.Args) + if err != nil { + t.Fatalf("Failed removing podman container.\n%s: %s", err, string(out)) + } + }) +} + +func podmanRun(t *testing.T, name, dockerRef, homeDir string, args ...string) { //nolint:unparam + t.Run(name, func(t *testing.T) { + cmdArgs := []string{"run", "-i", "--rm", "--privileged", "--network=host", dockerRef} + cmdArgs = append(cmdArgs, args...) + cmd := exec.Command("podman", cmdArgs...) + cmd.Env = append(cmd.Env, "HOME="+homeDir) + out, err := cmd.CombinedOutput() + t.Log(cmd.Args) + if err != nil { + t.Fatalf("Failed running podman container.\n%s: %s", err, string(out)) + } + }) +} + +// E2ETests is the main func to trigger the test suite +func E2ETests(env e2e.TestEnv) testhelper.Tests { + c := ctx{ + env: env, + } + + return testhelper.Tests{ + "Docker": c.docker, + "Porman": c.podman, + } +} diff --git a/e2e/suite.go b/e2e/suite.go index a13977a955..898d7bd286 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -1,5 +1,5 @@ // Copyright (c) 2020, Control Command Inc. All rights reserved. -// Copyright (c) 2019-2022 Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2025 Sylabs Inc. All rights reserved. // Copyright (c) Contributors to the Apptainer project, established as // Apptainer a Series of LF Projects LLC. // This software is licensed under a 3-clause BSD license. Please consult the @@ -40,6 +40,7 @@ import ( "github.com/sylabs/singularity/v4/e2e/instance" "github.com/sylabs/singularity/v4/e2e/key" "github.com/sylabs/singularity/v4/e2e/keyserver" + "github.com/sylabs/singularity/v4/e2e/nested" "github.com/sylabs/singularity/v4/e2e/oci" "github.com/sylabs/singularity/v4/e2e/overlay" "github.com/sylabs/singularity/v4/e2e/plugin" @@ -85,6 +86,7 @@ var e2eGroups = map[string]testhelper.Group{ "INSTANCE": instance.E2ETests, "KEY": key.E2ETests, "KEYSERVER": keyserver.E2ETests, + "NESTED": nested.E2ETests, "OCI": oci.E2ETests, "OVERLAY": overlay.E2ETests, "PLUGIN": plugin.E2ETests, diff --git a/e2e/testdata/Dockerfile.nested b/e2e/testdata/Dockerfile.nested new file mode 100644 index 0000000000..8df8c6f15b --- /dev/null +++ b/e2e/testdata/Dockerfile.nested @@ -0,0 +1,39 @@ +FROM ubuntu:24.10 + +ARG GOVERSION="go1.24.0" +ARG GOOS="linux" +ARG GOARCH="amd64" +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin +ENV DEBIAN_FRONTEND=noninteractive + +# Install system packages +RUN apt-get update && apt-get install -y autoconf \ + automake \ + crun \ + cryptsetup \ + fuse2fs \ + fuse \ + fuse-overlayfs \ + git \ + libfuse-dev \ + libglib2.0-dev \ + libseccomp-dev \ + libsubid-dev \ + libtool \ + make \ + pkg-config \ + squashfs-tools \ + squashfs-tools-ng \ + tzdata \ + uidmap \ + wget + +# Install GO +RUN wget -O /tmp/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz https://go.dev/dl/${GOVERSION}.${GOOS}-${GOARCH}.tar.gz && tar -C /usr/local -xzf /tmp/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz + +# Install SingularityCE +ADD . singularity +RUN cd singularity && git clean -fdx && ./mconfig && make -C builddir && make -C builddir install + +ENTRYPOINT ["/usr/local/bin/singularity"] +CMD ["version"] \ No newline at end of file