feat: chroot isolation

cmd/executor/cmd/root.go

@@ -223,6 +223,7 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().BoolVarP(&opts.CacheCopyLayers, "cache-copy-layers", "", false, "Caches copy layers")
 	RootCmd.PersistentFlags().VarP(&opts.IgnorePaths, "ignore-path", "", "Ignore these paths when taking a snapshot. Set it repeatedly for multiple paths.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.ForceBuildMetadata, "force-build-metadata", "", false, "Force add metadata layers to build image")
+	RootCmd.PersistentFlags().StringVarP(&opts.Isolation, "isolation", "", "chroot", "Which isolation method to use (possible implementation: chroot, none)")
 
 	// Allow setting --registry-mirror using an environment variable.
 	if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok {

deploy/Dockerfile

@@ -44,10 +44,37 @@ RUN \
 FROM debian:bullseye-slim AS certs
 RUN apt update && apt install -y ca-certificates
 
+FROM debian:bullseye-slim AS passwd
+RUN echo "root:x:0:" > /etc/group 
+RUN echo "root:x:0:0:root:/root:/sbin/nologin" > /etc/passwd
+
+# idmap runnable without --privileged (but still requires seccomp=unconfined apparmor=unconfined)
+FROM alpine AS idmap
+ARG SHADOW_VERSION="v4.11.1"
+ARG PATCHELF_VERSION="0.15.0"
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apk add --no-cache automake autoconf gettext git build-base libcap-dev libtool make gettext gettext-dev linux-pam-dev expect byacc
+RUN git clone https://github.com/shadow-maint/shadow.git /shadow
+WORKDIR /shadow
+RUN git fetch && git checkout $SHADOW_VERSION
+RUN  ./autogen.sh --disable-nls --disable-man --without-audit --without-selinux --without-acl --without-attr --without-tcb --without-nscd \
+  --disable-shared --enable-static=yes 
+RUN make 
+# set setgid and setuid filemode bits on new{gid,uid}map
+RUN chmod g+s src/newgidmap && chmod u+s src/newuidmap
+
+RUN wget -O patchelf.tar.gz https://github.com/NixOS/patchelf/releases/download/${PATCHELF_VERSION}/patchelf-${PATCHELF_VERSION}-$(uname -m).tar.gz && \
+  tar xf patchelf.tar.gz && \
+  cp bin/patchelf /bin
+
+# use patchelf to modify the library path so lib musl can reside in /kaniko/lib dir
+RUN patchelf --set-interpreter /kaniko/lib/ld-musl-$(uname -m).so.1 src/newuidmap
+RUN patchelf --set-interpreter /kaniko/lib/ld-musl-$(uname -m).so.1 src/newgidmap
+
 FROM scratch
 # Create kaniko directory with world write permission to allow non root run
 # use musl busybox since it's staticly compiled
-RUN --mount=from=busybox:musl,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] 
+RUN --mount=from=busybox:musl,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko /root /etc && chmod 777 /kaniko"] 
 
 COPY --from=0 /src/out/executor /kaniko/executor
 COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr
@@ -56,8 +83,12 @@ COPY --from=0 /usr/local/bin/docker-credential-acr-env /kaniko/docker-credential
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /kaniko/ssl/certs/
 COPY --from=0 /kaniko/.docker /kaniko/.docker
 COPY files/nsswitch.conf /etc/nsswitch.conf
-ENV HOME /root
-ENV USER root
+COPY --from=passwd /etc/passwd /etc/group /etc/
+COPY --from=idmap /shadow/src/newuidmap /shadow/src/newgidmap /kaniko/
+# shadowutils is not a static binary, need to use musl libc
+COPY --from=idmap /lib/ld-musl-*.so.1 /lib/libc.musl-*.so.1 /kaniko/lib/
+
+USER root
 ENV PATH /usr/local/bin:/kaniko
 ENV SSL_CERT_DIR=/kaniko/ssl/certs
 ENV DOCKER_CONFIG /kaniko/.docker/

deploy/Dockerfile_debug

@@ -45,11 +45,40 @@ RUN \
 FROM debian:bullseye-slim AS certs
 RUN apt update && apt install -y ca-certificates
 
+
+FROM debian:bullseye-slim AS passwd
+RUN echo "root:x:0:" > /etc/group 
+RUN echo "root:x:0:0:root:/root:/sbin/nologin" > /etc/passwd
+
+# idmap runnable without --privileged (but still requires seccomp=unconfined apparmor=unconfined)
+FROM alpine AS idmap
+ARG SHADOW_VERSION="v4.11.1"
+ARG PATCHELF_VERSION="0.15.0"
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apk add --no-cache automake autoconf gettext git build-base libcap-dev libtool make gettext gettext-dev linux-pam-dev expect byacc
+RUN git clone https://github.com/shadow-maint/shadow.git /shadow
+WORKDIR /shadow
+RUN git fetch && git checkout $SHADOW_VERSION
+RUN  ./autogen.sh --disable-nls --disable-man --without-audit --without-selinux --without-acl --without-attr --without-tcb --without-nscd \
+  --disable-shared --enable-static=yes 
+RUN make
+# set setgid and setuid filemode bits on new{gid,uid}map
+RUN chmod g+s src/newgidmap && chmod u+s src/newuidmap
+
+RUN wget -O patchelf.tar.gz https://github.com/NixOS/patchelf/releases/download/${PATCHELF_VERSION}/patchelf-${PATCHELF_VERSION}-$(uname -m).tar.gz && \
+  tar xf patchelf.tar.gz && \
+  cp bin/patchelf /bin
+
+# use patchelf to modify the library path so lib musl can reside in /kaniko/lib dir
+RUN patchelf --set-interpreter /kaniko/lib/ld-musl-$(uname -m).so.1 src/newuidmap
+RUN patchelf --set-interpreter /kaniko/lib/ld-musl-$(uname -m).so.1 src/newgidmap
+
+
 # use musl busybox since it's staticly compiled on all platforms
 FROM busybox:musl as busybox
 FROM scratch
 # Create kaniko directory with world write permission to allow non root run
-RUN --mount=from=busybox,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] 
+RUN --mount=from=busybox,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko /root && chmod 777 /kaniko"] 
 
 COPY --from=0 /src/out/executor /kaniko/executor
 COPY --from=0 /src/out/warmer /kaniko/warmer
@@ -63,8 +92,12 @@ VOLUME /busybox
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /kaniko/ssl/certs/
 COPY --from=0 /kaniko/.docker /kaniko/.docker
 COPY files/nsswitch.conf /etc/nsswitch.conf
-ENV HOME /root
-ENV USER root
+COPY --from=passwd /etc/passwd /etc/group /etc/
+COPY --from=idmap /shadow/src/newuidmap /shadow/src/newgidmap /kaniko/
+# shadowutils is not a static binary, need to use musl libc
+COPY --from=idmap /lib/ld-musl-*.so.1 /lib/libc.musl-*.so.1 /kaniko/lib/
+
+USER root
 ENV PATH /usr/local/bin:/kaniko:/busybox
 ENV SSL_CERT_DIR=/kaniko/ssl/certs
 ENV DOCKER_CONFIG /kaniko/.docker/

go.mod

@@ -118,7 +118,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/moby/locker v1.0.1 // indirect
-	github.com/moby/sys/mount v0.3.0 // indirect
+	github.com/moby/sys/mount v0.3.0
 	github.com/moby/sys/mountinfo v0.5.0 // indirect
 	github.com/moby/sys/symlink v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect

integration/dockerfiles-isolation/Dockerfile_test_chroot

@@ -0,0 +1,3 @@
+FROM alpine
+
+RUN ls -al

integration/dockerfiles/Dockerfile_test_user_nonexisting

@@ -14,6 +14,7 @@
 
 FROM debian:9.11
 USER 1001:1001
+RUN ["echo", "hello"]
 RUN echo "hey2" > /tmp/foo
 USER 1001
 RUN echo "hello" > /tmp/foobar

integration/images.go

@@ -238,6 +238,17 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string {
 	return flags
 }
 
+// addSecurityFlags adds seccomp and apparmor profile disable and adds cap SYS_ADMIN
+func addSecurityFlags(flags []string) []string {
+	return append(flags, []string{
+		// chroot needs CAP_SYS_ADMIN
+		"--cap-add", "SYS_ADMIN",
+		// disable apparmor and seccomp because it permits mounts
+		"--security-opt", "apparmor=unconfined",
+		"--security-opt", "seccomp=unconfined",
+	}...)
+}
+
 func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfilesPath, dockerfile, contextDir string) error {
 	t.Logf("Building image for Dockerfile %s\n", dockerfile)
 
@@ -283,7 +294,6 @@ func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfile
 func (d *DockerFileBuilder) BuildImage(t *testing.T, config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
 	_, ex, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(ex)
-
 	return d.BuildImageWithContext(t, config, dockerfilesPath, dockerfile, cwd)
 }
 
@@ -375,16 +385,17 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac
 			"-v", cwd + ":/workspace",
 			"-e", benchmarkEnv}
 		dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
+		dockerRunFlags = addSecurityFlags(dockerRunFlags)
+
 		dockerRunFlags = append(dockerRunFlags, ExecutorImage,
 			"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
 			"-d", kanikoImage,
 			"-c", buildContextPath,
 			cacheFlag,
 			"--cache-repo", cacheRepo,
-			"--cache-dir", cacheDir)
-		for _, v := range args {
-			dockerRunFlags = append(dockerRunFlags, v)
-		}
+			"--cache-dir", cacheDir,
+		)
+		dockerRunFlags = append(dockerRunFlags, args...)
 		kanikoCmd := exec.Command("docker", dockerRunFlags...)
 
 		_, err := RunCommandWithoutTest(kanikoCmd)
@@ -420,6 +431,7 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi
 
 	dockerRunFlags := []string{"run", "--net=host", "-v", cwd + ":/workspace"}
 	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
+	dockerRunFlags = addSecurityFlags(dockerRunFlags)
 	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
 		"-f", dockerfile,
 		"-d", kanikoImage,
@@ -481,15 +493,15 @@ func buildKanikoImage(
 		"-v", contextDir + ":/workspace",
 		"-v", benchmarkDir + ":/kaniko/benchmarks",
 	}
+	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
+	dockerRunFlags = addSecurityFlags(dockerRunFlags)
 
 	if env, ok := envsMap[dockerfile]; ok {
 		for _, envVariable := range env {
 			dockerRunFlags = append(dockerRunFlags, "-e", envVariable)
 		}
 	}
 
-	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
-
 	kanikoDockerfilePath := path.Join(buildContextPath, dockerfilesPath, dockerfile)
 	if dockerfilesPath == "" {
 		kanikoDockerfilePath = path.Join(buildContextPath, "Dockerfile")

integration/integration_test.go

@@ -262,6 +262,7 @@ func testGitBuildcontextHelper(t *testing.T, repo string) {
 	kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git")
 	dockerRunFlags := []string{"run", "--net=host"}
 	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = addSecurityFlags(dockerRunFlags)
 	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
 		"-f", dockerfile,
 		"-d", kanikoImage,
@@ -327,6 +328,7 @@ func TestGitBuildcontextSubPath(t *testing.T) {
 	kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git")
 	dockerRunFlags := []string{"run", "--net=host"}
 	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = addSecurityFlags(dockerRunFlags)
 	dockerRunFlags = append(
 		dockerRunFlags,
 		ExecutorImage,
@@ -369,6 +371,7 @@ func TestBuildViaRegistryMirrors(t *testing.T) {
 	kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_registry_mirror")
 	dockerRunFlags := []string{"run", "--net=host"}
 	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = addSecurityFlags(dockerRunFlags)
 	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
 		"-f", dockerfile,
 		"-d", kanikoImage,
@@ -410,6 +413,7 @@ func TestKanikoDir(t *testing.T) {
 	kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_registry_mirror")
 	dockerRunFlags := []string{"run", "--net=host"}
 	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = addSecurityFlags(dockerRunFlags)
 	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
 		"-f", dockerfile,
 		"-d", kanikoImage,
@@ -452,6 +456,7 @@ func TestBuildWithLabels(t *testing.T) {
 	kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_label:mylabel")
 	dockerRunFlags := []string{"run", "--net=host"}
 	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = addSecurityFlags(dockerRunFlags)
 	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
 		"-f", dockerfile,
 		"-d", kanikoImage,
@@ -492,6 +497,7 @@ func TestBuildWithHTTPError(t *testing.T) {
 	kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_add_404")
 	dockerRunFlags := []string{"run", "--net=host"}
 	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = addSecurityFlags(dockerRunFlags)
 	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
 		"-f", dockerfile,
 		"-d", kanikoImage,
@@ -668,6 +674,7 @@ func TestExitCodePropagation(t *testing.T) {
 			"-v", contextVolume,
 		}
 		dockerFlags = addServiceAccountFlags(dockerFlags, "")
+		dockerFlags = addSecurityFlags(dockerFlags)
 		dockerFlags = append(dockerFlags, ExecutorImage,
 			"-c", "dir:///workspace/",
 			"-f", "./Dockerfile_exit_code_propagation",

integration/integration_with_stdin_test.go

@@ -71,7 +71,7 @@ func TestBuildWithStdin(t *testing.T) {
 		gw := gzip.NewWriter(tarFile)
 		defer gw.Close()
 
-		tw := util.NewTar(gw)
+		tw := util.NewTar(testDirLongPath, gw)
 		defer tw.Close()
 
 		if err := tw.AddFileToTar(dockerfile); err != nil {
@@ -103,6 +103,7 @@ func TestBuildWithStdin(t *testing.T) {
 
 	dockerRunFlags := []string{"run", "--interactive", "--net=host", "-v", cwd + ":/workspace"}
 	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = addSecurityFlags(dockerRunFlags)
 	dockerRunFlags = append(dockerRunFlags,
 		ExecutorImage,
 		"-f", dockerfile,

integration/k8s-job.yaml

@@ -4,6 +4,11 @@ metadata:
   name: kaniko-test-{{.Name}}
 spec:
   template:
+    metadata:
+      annotations:
+        # disable seccomp and apparmor for chroot mounts
+        container.apparmor.security.beta.kubernetes.io/kaniko: unconfined 
+        container.seccomp.security.alpha.kubernetes.io/kaniko: unconfined
     spec:
       hostNetwork: true
       containers:
@@ -15,6 +20,10 @@ spec:
         volumeMounts:
         - name: context
           mountPath: /workspace
+        securityContext:
+          capabilities:
+            add:
+              - SYS_ADMIN
       restartPolicy: Never
       volumes:
       - name: context

integration/tar.go

@@ -49,7 +49,7 @@ func CreateIntegrationTarball() (string, error) {
 	gzipWriter := gzip.NewWriter(file)
 	defer gzipWriter.Close()
 
-	err = util.CreateTarballOfDirectory(dir, file)
+	err = util.CreateTarballOfDirectory("/", dir, file)
 	if err != nil {
 		return "", fmt.Errorf("creating tarball of integration dir: %w", err)
 	}

pkg/buildcontext/tar_test.go

@@ -84,7 +84,7 @@ func TestBuildWithLocalTar(t *testing.T) {
 		gw := gzip.NewWriter(validTarFile)
 		defer gw.Close()
 
-		tw := util.NewTar(gw)
+		tw := util.NewTar(testDirLongPath, gw)
 		defer tw.Close()
 
 		if err := tw.AddFileToTar(validDockerfile); err != nil {

pkg/chroot/capabilities_linux.go

@@ -0,0 +1,49 @@
+//go:build linux
+// +build linux
+
+package chroot
+
+import (
+	"fmt"
+
+	"github.com/syndtr/gocapability/capability"
+)
+
+// defaultCapabilities returns a Linux kernel default capabilities
+var defaultCapabilities = []capability.Cap{
+	capability.CAP_CHOWN,
+	capability.CAP_DAC_OVERRIDE,
+	capability.CAP_FSETID,
+	capability.CAP_FOWNER,
+	capability.CAP_MKNOD,
+	capability.CAP_NET_RAW,
+	capability.CAP_SETGID,
+	capability.CAP_SETUID,
+	capability.CAP_SETFCAP,
+	capability.CAP_SETPCAP,
+	capability.CAP_NET_BIND_SERVICE,
+	capability.CAP_KILL,
+	capability.CAP_AUDIT_WRITE,
+}
+
+// setCapabilities sets capabilities for ourselves, to be more or less inherited by any processes that we'll start.
+func setCapabilities() error {
+	caps, err := capability.NewPid2(0)
+	if err != nil {
+		return err
+	}
+	capMap := map[capability.CapType][]capability.Cap{
+		capability.BOUNDING:    defaultCapabilities,
+		capability.EFFECTIVE:   defaultCapabilities,
+		capability.INHERITABLE: {},
+		capability.PERMITTED:   defaultCapabilities,
+	}
+	for capType, capList := range capMap {
+		caps.Set(capType, capList...)
+	}
+	err = caps.Apply(capability.CAPS | capability.BOUNDS | capability.AMBS)
+	if err != nil {
+		return fmt.Errorf("applying capabiliies: %w", err)
+	}
+	return nil
+}