Add support for cgroup namespace

Signed-off-by: Yuanhong Peng <[email protected]>

libcontainer/README.md

@@ -71,6 +71,7 @@ config := &configs.Config{
 		{Type: configs.NEWPID},
 		{Type: configs.NEWUSER},
 		{Type: configs.NEWNET},
+		{Type: configs.NEWCGROUP},
 	}),
 	Cgroups: &configs.Cgroup{
 		Name:   "test-container",

libcontainer/SPEC.md

@@ -21,16 +21,17 @@ Minimum requirements:
 
 ### Namespaces
 
-|     Flag      | Enabled | 
-| ------------  | ------- |
-| CLONE_NEWPID  |    1    |
-| CLONE_NEWUTS  |    1    |
-| CLONE_NEWIPC  |    1    |
-| CLONE_NEWNET  |    1    |
-| CLONE_NEWNS   |    1    |
-| CLONE_NEWUSER |    1    |
-
-Namespaces are created for the container via the `clone` syscall.  
+|     Flag        | Enabled |
+| --------------- | ------- |
+| CLONE_NEWPID    |    1    |
+| CLONE_NEWUTS    |    1    |
+| CLONE_NEWIPC    |    1    |
+| CLONE_NEWNET    |    1    |
+| CLONE_NEWNS     |    1    |
+| CLONE_NEWUSER   |    1    |
+| CLONE_NEWCGROUP |    1    |
+
+Namespaces are created for the container via the `unshare` syscall.
 
 
 ### Filesystem

libcontainer/cgroups/utils.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	cgroupNamePrefix = "name="
+	CgroupNamePrefix = "name="
 	CgroupProcesses  = "cgroup.procs"
 )
 
@@ -161,8 +161,8 @@ func getCgroupMountsHelper(ss map[string]bool, mi io.Reader, all bool) ([]Mount,
 			if !ss[opt] {
 				continue
 			}
-			if strings.HasPrefix(opt, cgroupNamePrefix) {
-				m.Subsystems = append(m.Subsystems, opt[len(cgroupNamePrefix):])
+			if strings.HasPrefix(opt, CgroupNamePrefix) {
+				m.Subsystems = append(m.Subsystems, opt[len(CgroupNamePrefix):])
 			} else {
 				m.Subsystems = append(m.Subsystems, opt)
 			}
@@ -316,7 +316,7 @@ func getControllerPath(subsystem string, cgroups map[string]string) (string, err
 		return p, nil
 	}
 
-	if p, ok := cgroups[cgroupNamePrefix+subsystem]; ok {
+	if p, ok := cgroups[CgroupNamePrefix+subsystem]; ok {
 		return p, nil
 	}
 

libcontainer/configs/namespaces_syscall.go

@@ -9,7 +9,7 @@ func (n *Namespace) Syscall() int {
 }
 
 // This is not yet in the Go stdlib.
-const syscall_CLONE_NEWCGROUP = (1 << 29)
+const Syscall_CLONE_NEWCGROUP = (1 << 25)
 
 var namespaceInfo = map[NamespaceType]int{
 	NEWNET:    syscall.CLONE_NEWNET,
@@ -18,7 +18,7 @@ var namespaceInfo = map[NamespaceType]int{
 	NEWIPC:    syscall.CLONE_NEWIPC,
 	NEWUTS:    syscall.CLONE_NEWUTS,
 	NEWPID:    syscall.CLONE_NEWPID,
-	NEWCGROUP: syscall_CLONE_NEWCGROUP,
+	NEWCGROUP: Syscall_CLONE_NEWCGROUP,
 }
 
 // CloneFlags parses the container's Namespaces options to set the correct

libcontainer/container_linux.go

@@ -1234,6 +1234,7 @@ func (c *linuxContainer) orderNamespacePaths(namespaces map[configs.NamespaceTyp
 		configs.NEWNET,
 		configs.NEWPID,
 		configs.NEWNS,
+		configs.NEWCGROUP,
 	}
 
 	// Remove namespaces that we don't need to join.

libcontainer/integration/exec_test.go

@@ -1660,3 +1660,54 @@ func TestTmpfsCopyUp(t *testing.T) {
 		t.Fatalf("/etc/passwd not copied up as expected: %v", outputLs)
 	}
 }
+
+func TestCGROUPPrivate(t *testing.T) {
+	if testing.Short() {
+		return
+	}
+
+	rootfs, err := newRootfs()
+	ok(t, err)
+	defer remove(rootfs)
+
+	l, err := os.Readlink("/proc/1/ns/cgroup")
+	ok(t, err)
+
+	config := newTemplateConfig(rootfs)
+	config.Namespaces.Add(configs.NEWCGROUP, "")
+	buffers, exitCode, err := runContainer(config, "", "readlink", "/proc/self/ns/cgroup")
+	ok(t, err)
+
+	if exitCode != 0 {
+		t.Fatalf("exit code not 0. code %d stderr %q", exitCode, buffers.Stderr)
+	}
+
+	if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual == l {
+		t.Fatalf("cgroup link should be private to the container but equals host %q %q", actual, l)
+	}
+}
+
+func TestCGROUPHost(t *testing.T) {
+	if testing.Short() {
+		return
+	}
+
+	rootfs, err := newRootfs()
+	ok(t, err)
+	defer remove(rootfs)
+
+	l, err := os.Readlink("/proc/1/ns/cgroup")
+	ok(t, err)
+
+	config := newTemplateConfig(rootfs)
+	buffers, exitCode, err := runContainer(config, "", "readlink", "/proc/self/ns/cgroup")
+	ok(t, err)
+
+	if exitCode != 0 {
+		t.Fatalf("exit code not 0. code %d stderr %q", exitCode, buffers.Stderr)
+	}
+
+	if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual != l {
+		t.Fatalf("cgroup link not equal to host link %q %q", actual, l)
+	}
+}

libcontainer/nsenter/nsexec.c

@@ -38,6 +38,9 @@ enum sync_t {
 	SYNC_ERR = 0xFF, /* Fatal error, no turning back. The error code follows. */
 };
 
+/* Synchronisation value for cgroup namespace setup */
+#define SYNC_PRIVATE_CGNS 0x80
+
 /* longjmp() arguments. */
 #define JUMP_PARENT 0x00
 #define JUMP_CHILD  0xA0
@@ -46,6 +49,9 @@ enum sync_t {
 /* JSON buffer. */
 #define JSON_MAX 4096
 
+/* buffer for synchronisation value */
+#define SYNC_BUF_LEN 10
+
 /* Assume the stack grows down, so arguments should be above it. */
 struct clone_t {
 	/*
@@ -608,6 +614,7 @@ void nsexec(void)
 	case JUMP_CHILD: {
 			pid_t child;
 			enum sync_t s;
+			uint32_t actual_flags = config.cloneflags;
 
 			/* We're in a child and thus need to tell the parent if we die. */
 			syncfd = syncpipe[0];
@@ -634,7 +641,10 @@ void nsexec(void)
 			 * some old kernel versions where clone(CLONE_PARENT | CLONE_NEWPID)
 			 * was broken, so we'll just do it the long way anyway.
 			 */
-			if (unshare(config.cloneflags) < 0)
+			if (actual_flags & CLONE_NEWCGROUP) {
+				actual_flags &= ~CLONE_NEWCGROUP;
+			}
+			if (unshare(actual_flags) < 0)
 				bail("failed to unshare namespaces");
 
 			/*
@@ -711,6 +721,9 @@ void nsexec(void)
 			 * start_child() code after forking in the parent.
 			 */
 			int consolefd = config.consolefd;
+			char buf[SYNC_BUF_LEN];
+			int len, value;
+			char *endptr;
 
 			/* We're in a child and thus need to tell the parent if we die. */
 			syncfd = syncpipe[0];
@@ -741,6 +754,22 @@ void nsexec(void)
 					bail("failed to dup stderr");
 			}
 
+			/* ... wait until our topmost parent has finished cgroup setup in p.manager.Apply() ... */
+			if (config.cloneflags & CLONE_NEWCGROUP) {
+				len = read(pipenum, buf, SYNC_BUF_LEN);
+				if (len < 0)
+					bail("read synchronisation value failed");
+				value = strtol(buf, &endptr, 10);
+				if (*endptr != '\0')
+					bail("unable to parse synchronisation value");
+				if (value == SYNC_PRIVATE_CGNS) {
+					if (unshare(CLONE_NEWCGROUP) < 0)
+						bail("failed to unshare cgroup namespace");
+				}
+				else
+					bail("received unknown synchronisation value");
+			}
+
 			/* Close sync pipes. */
 			close(syncpipe[0]);
 			close(syncpipe[1]);

libcontainer/process_linux.go

@@ -19,6 +19,9 @@ import (
 	"github.com/opencontainers/runc/libcontainer/utils"
 )
 
+// Synchronisation value for cgroup namespace setup.
+const privateCgroupns int = (1 << 7)
+
 type parentProcess interface {
 	// pid returns the pid for the running process.
 	pid() int
@@ -257,6 +260,13 @@ func (p *initProcess) start() error {
 	if err := p.manager.Apply(p.pid()); err != nil {
 		return newSystemErrorWithCause(err, "applying cgroup configuration for process")
 	}
+	// Now it's time to setup cgroup namesapce
+	if p.config.Config.Namespaces.Contains(configs.NEWCGROUP) && p.config.Config.Namespaces.PathOf(configs.NEWCGROUP) == "" {
+		if _, err := p.parentPipe.WriteString(strconv.Itoa(privateCgroupns)); err != nil {
+			return newSystemErrorWithCause(err, "sending synchronization value to init process")
+		}
+
+	}
 	defer func() {
 		if err != nil {
 			// TODO: should not be the responsibility to call here

libcontainer/rootfs_linux.go

@@ -43,14 +43,15 @@ func setupRootfs(config *configs.Config, console *linuxConsole, pipe io.ReadWrit
 		return newSystemErrorWithCause(err, "preparing rootfs")
 	}
 
+	hasCgroupns := config.Namespaces.Contains(configs.NEWCGROUP)
 	setupDev := needsSetupDev(config)
 	for _, m := range config.Mounts {
 		for _, precmd := range m.PremountCmds {
 			if err := mountCmd(precmd); err != nil {
 				return newSystemErrorWithCause(err, "running premount command")
 			}
 		}
-		if err := mountToRootfs(m, config.Rootfs, config.MountLabel); err != nil {
+		if err := mountToRootfs(m, config.Rootfs, config.MountLabel, hasCgroupns); err != nil {
 			return newSystemErrorWithCausef(err, "mounting %q to rootfs %q at %q", m.Source, config.Rootfs, m.Destination)
 		}
 
@@ -125,7 +126,7 @@ func mountCmd(cmd configs.Command) error {
 	return nil
 }
 
-func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error {
+func mountToRootfs(m *configs.Mount, rootfs, mountLabel string, enableCgroupns bool) error {
 	var (
 		dest = m.Destination
 	)
@@ -257,12 +258,33 @@ func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error {
 			Data:             "mode=755",
 			PropagationFlags: m.PropagationFlags,
 		}
-		if err := mountToRootfs(tmpfs, rootfs, mountLabel); err != nil {
+		if err := mountToRootfs(tmpfs, rootfs, mountLabel, enableCgroupns); err != nil {
 			return err
 		}
 		for _, b := range binds {
-			if err := mountToRootfs(b, rootfs, mountLabel); err != nil {
-				return err
+			if enableCgroupns {
+				subsystemPath := filepath.Join(rootfs, b.Destination)
+				if err := os.MkdirAll(subsystemPath, 0755); err != nil {
+					return err
+				}
+				flags := defaultMountFlags
+				if m.Flags&syscall.MS_RDONLY != 0 {
+					flags = flags | syscall.MS_RDONLY
+				}
+				cgroupmount := &configs.Mount{
+					Source:      "cgroup",
+					Device:      "cgroup",
+					Destination: subsystemPath,
+					Flags:       flags,
+					Data:        filepath.Base(subsystemPath),
+				}
+				if err := mountNewCgroup(cgroupmount); err != nil {
+					return err
+				}
+			} else {
+				if err := mountToRootfs(b, rootfs, mountLabel, enableCgroupns); err != nil {
+					return err
+				}
 			}
 		}
 		for _, mc := range merged {
@@ -764,3 +786,18 @@ func mountPropagate(m *configs.Mount, rootfs string, mountLabel string) error {
 	}
 	return nil
 }
+
+func mountNewCgroup(m *configs.Mount) error {
+	var (
+		data   = m.Data
+		source = m.Source
+	)
+	if data == "systemd" {
+		data = cgroups.CgroupNamePrefix + data
+		source = "systemd"
+	}
+	if err := syscall.Mount(source, m.Destination, m.Device, uintptr(m.Flags), data); err != nil {
+		return err
+	}
+	return nil
+}

libcontainer/specconv/spec_linux.go

@@ -183,6 +183,10 @@ func CreateLibcontainerConfig(opts *CreateOpts) (*configs.Config, error) {
 		return nil, fmt.Errorf("rootfsPropagation=%v is not supported", spec.Linux.RootfsPropagation)
 	}
 
+	cgroupPathOfHost, err := os.Readlink("/proc/1/ns/cgroup")
+	if err != nil {
+		return nil, err
+	}
 	for _, ns := range spec.Linux.Namespaces {
 		t, exists := namespaceMapping[ns.Type]
 		if !exists {
@@ -191,6 +195,17 @@ func CreateLibcontainerConfig(opts *CreateOpts) (*configs.Config, error) {
 		if config.Namespaces.Contains(t) {
 			return nil, fmt.Errorf("malformed spec file: duplicated ns %q", ns)
 		}
+		// if this container is going to have shared cgroup namespace with host,
+		// we just skip the following .Add()
+		if t == configs.NEWCGROUP && ns.Path != "" {
+			cgroupPathOfContainer, err := os.Readlink(ns.Path)
+			if err != nil {
+				return nil, err
+			}
+			if strings.Compare(cgroupPathOfHost, cgroupPathOfContainer) == 0 {
+				continue
+			}
+		}
 		config.Namespaces.Add(t, ns.Path)
 	}
 	if config.Namespaces.Contains(configs.NEWNET) {