Merge pull request #3754 from filipochnik/docker-caps

Add an option to add and drop capabilities in the Docker driver

Author: dadgar

client/driver/docker.go

@@ -99,6 +99,11 @@ const (
 	dockerImageRemoveDelayConfigOption  = "docker.cleanup.image.delay"
 	dockerImageRemoveDelayConfigDefault = 3 * time.Minute
 
+	// dockerCapsWhitelistConfigOption is the key for setting the list of
+	// allowed Linux capabilities
+	dockerCapsWhitelistConfigOption  = "docker.caps.whitelist"
+	dockerCapsWhitelistConfigDefault = dockerBasicCaps
+
 	// dockerTimeout is the length of time a request can be outstanding before
 	// it is timed out.
 	dockerTimeout = 5 * time.Minute
@@ -109,6 +114,12 @@ const (
 	// dockerAuthHelperPrefix is the prefix to attach to the credential helper
 	// and should be found in the $PATH. Example: ${prefix-}${helper-name}
 	dockerAuthHelperPrefix = "docker-credential-"
+
+	// dockerBasicCaps is comma-separated list of Linux capabilities that are
+	// allowed by docker by default, as documented in
+	// https://docs.docker.com/engine/reference/run/#block-io-bandwidth-blkio-constraint
+	dockerBasicCaps = "CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID," +
+		"SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE"
 )
 
 type DockerDriver struct {
@@ -202,6 +213,8 @@ type DockerDriverConfig struct {
 	MacAddress       string              `mapstructure:"mac_address"`        // Pin mac address to container
 	SecurityOpt      []string            `mapstructure:"security_opt"`       // Flags to pass directly to security-opt
 	Devices          []DockerDevice      `mapstructure:"devices"`            // To allow mounting USB or other serial control devices
+	CapAdd           []string            `mapstructure:"cap_add"`            // Flags to pass directly to cap-add
+	CapDrop          []string            `mapstructure:"cap_drop"`           // Flags to pass directly to cap-drop
 }
 
 func sliceMergeUlimit(ulimitsRaw map[string]string) ([]docker.ULimit, error) {
@@ -304,6 +317,8 @@ func NewDockerDriverConfig(task *structs.Task, env *env.TaskEnv) (*DockerDriverC
 	dconf.ExtraHosts = env.ParseAndReplace(dconf.ExtraHosts)
 	dconf.MacAddress = env.ReplaceEnv(dconf.MacAddress)
 	dconf.SecurityOpt = env.ParseAndReplace(dconf.SecurityOpt)
+	dconf.CapAdd = env.ParseAndReplace(dconf.CapAdd)
+	dconf.CapDrop = env.ParseAndReplace(dconf.CapDrop)
 
 	for _, m := range dconf.SysctlRaw {
 		for k, v := range m {
@@ -644,6 +659,12 @@ func (d *DockerDriver) Validate(config map[string]interface{}) error {
 			"devices": {
 				Type: fields.TypeArray,
 			},
+			"cap_add": {
+				Type: fields.TypeArray,
+			},
+			"cap_drop": {
+				Type: fields.TypeArray,
+			},
 		},
 	}
 
@@ -1115,6 +1136,39 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas
 	}
 	hostConfig.Privileged = driverConfig.Privileged
 
+	// set capabilities
+	hostCapsWhitelistConfig := d.config.ReadDefault(
+		dockerCapsWhitelistConfigOption, dockerCapsWhitelistConfigDefault)
+	hostCapsWhitelist := make(map[string]struct{})
+	for _, cap := range strings.Split(hostCapsWhitelistConfig, ",") {
+		cap = strings.ToLower(strings.TrimSpace(cap))
+		hostCapsWhitelist[cap] = struct{}{}
+	}
+
+	if _, ok := hostCapsWhitelist["all"]; !ok {
+		effectiveCaps, err := tweakCapabilities(
+			strings.Split(dockerBasicCaps, ","),
+			driverConfig.CapAdd,
+			driverConfig.CapDrop,
+		)
+		if err != nil {
+			return c, err
+		}
+		var missingCaps []string
+		for _, cap := range effectiveCaps {
+			cap = strings.ToLower(cap)
+			if _, ok := hostCapsWhitelist[cap]; !ok {
+				missingCaps = append(missingCaps, cap)
+			}
+		}
+		if len(missingCaps) > 0 {
+			return c, fmt.Errorf("Docker driver doesn't have the following caps whitelisted on this Nomad agent: %s", missingCaps)
+		}
+	}
+
+	hostConfig.CapAdd = driverConfig.CapAdd
+	hostConfig.CapDrop = driverConfig.CapDrop
+
 	// set SHM size
 	if driverConfig.ShmSize != 0 {
 		hostConfig.ShmSize = driverConfig.ShmSize

client/driver/docker_default.go

@@ -2,7 +2,10 @@
 
 package driver
 
-import docker "github.com/fsouza/go-dockerclient"
+import (
+	docker "github.com/fsouza/go-dockerclient"
+	"github.com/moby/moby/daemon/caps"
+)
 
 const (
 	// Setting default network mode for non-windows OS as bridge
@@ -12,3 +15,22 @@ const (
 func getPortBinding(ip string, port string) []docker.PortBinding {
 	return []docker.PortBinding{{HostIP: ip, HostPort: port}}
 }
+
+func tweakCapabilities(basics, adds, drops []string) ([]string, error) {
+	// Moby mixes 2 different capabilities formats: prefixed with "CAP_"
+	// and not. We do the conversion here to have a consistent,
+	// non-prefixed format on the Nomad side.
+	for i, cap := range basics {
+		basics[i] = "CAP_" + cap
+	}
+
+	effectiveCaps, err := caps.TweakCapabilities(basics, adds, drops)
+	if err != nil {
+		return effectiveCaps, err
+	}
+
+	for i, cap := range effectiveCaps {
+		effectiveCaps[i] = cap[len("CAP_"):]
+	}
+	return effectiveCaps, nil
+}

client/driver/docker_test.go

@@ -1048,6 +1048,130 @@ func TestDockerDriver_SecurityOpt(t *testing.T) {
 	}
 }
 
+func TestDockerDriver_Capabilities(t *testing.T) {
+	if !tu.IsTravis() {
+		t.Parallel()
+	}
+	if !testutil.DockerIsConnected(t) {
+		t.Skip("Docker not connected")
+	}
+	if runtime.GOOS == "windows" {
+		t.Skip("Capabilities not supported on windows")
+	}
+
+	testCases := []struct {
+		Name       string
+		CapAdd     []string
+		CapDrop    []string
+		Whitelist  string
+		StartError string
+	}{
+		{
+			Name:    "default-whitelist-add-allowed",
+			CapAdd:  []string{"fowner", "mknod"},
+			CapDrop: []string{"all"},
+		},
+		{
+			Name:       "default-whitelist-add-forbidden",
+			CapAdd:     []string{"net_admin"},
+			StartError: "net_admin",
+		},
+		{
+			Name:    "default-whitelist-drop-existing",
+			CapDrop: []string{"fowner", "mknod"},
+		},
+		{
+			Name:      "restrictive-whitelist-drop-all",
+			CapDrop:   []string{"all"},
+			Whitelist: "fowner,mknod",
+		},
+		{
+			Name:      "restrictive-whitelist-add-allowed",
+			CapAdd:    []string{"fowner", "mknod"},
+			CapDrop:   []string{"all"},
+			Whitelist: "fowner,mknod",
+		},
+		{
+			Name:       "restrictive-whitelist-add-forbidden",
+			CapAdd:     []string{"net_admin", "mknod"},
+			CapDrop:    []string{"all"},
+			Whitelist:  "fowner,mknod",
+			StartError: "net_admin",
+		},
+		{
+			Name:      "permissive-whitelist",
+			CapAdd:    []string{"net_admin", "mknod"},
+			Whitelist: "all",
+		},
+		{
+			Name:      "permissive-whitelist-add-all",
+			CapAdd:    []string{"all"},
+			Whitelist: "all",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			client := newTestDockerClient(t)
+			task, _, _ := dockerTask(t)
+			if len(tc.CapAdd) > 0 {
+				task.Config["cap_add"] = tc.CapAdd
+			}
+			if len(tc.CapDrop) > 0 {
+				task.Config["cap_drop"] = tc.CapDrop
+			}
+
+			tctx := testDockerDriverContexts(t, task)
+			if tc.Whitelist != "" {
+				tctx.DriverCtx.config.Options[dockerCapsWhitelistConfigOption] = tc.Whitelist
+			}
+
+			driver := NewDockerDriver(tctx.DriverCtx)
+			copyImage(t, tctx.ExecCtx.TaskDir, "busybox.tar")
+			defer tctx.AllocDir.Destroy()
+
+			presp, err := driver.Prestart(tctx.ExecCtx, task)
+			defer driver.Cleanup(tctx.ExecCtx, presp.CreatedResources)
+			if err != nil {
+				t.Fatalf("Error in prestart: %v", err)
+			}
+
+			sresp, err := driver.Start(tctx.ExecCtx, task)
+			if err == nil && tc.StartError != "" {
+				t.Fatalf("Expected error in start: %v", tc.StartError)
+			} else if err != nil {
+				if tc.StartError == "" {
+					t.Fatalf("Failed to start driver: %s\nStack\n%s", err, debug.Stack())
+				} else if !strings.Contains(err.Error(), tc.StartError) {
+					t.Fatalf("Expect error containing \"%s\", got %v", tc.StartError, err)
+				}
+				return
+			}
+
+			if sresp.Handle == nil {
+				t.Fatalf("handle is nil\nStack\n%s", debug.Stack())
+			}
+			defer sresp.Handle.Kill()
+			handle := sresp.Handle.(*DockerHandle)
+
+			waitForExist(t, client, handle)
+
+			container, err := client.InspectContainer(handle.ContainerID())
+			if err != nil {
+				t.Fatalf("Error inspecting container: %v", err)
+			}
+
+			if !reflect.DeepEqual(tc.CapAdd, container.HostConfig.CapAdd) {
+				t.Errorf("CapAdd doesn't match.\nExpected:\n%s\nGot:\n%s\n", tc.CapAdd, container.HostConfig.CapAdd)
+			}
+
+			if !reflect.DeepEqual(tc.CapDrop, container.HostConfig.CapDrop) {
+				t.Errorf("CapDrop doesn't match.\nExpected:\n%s\nGot:\n%s\n", tc.CapDrop, container.HostConfig.CapDrop)
+			}
+		})
+	}
+}
+
 func TestDockerDriver_DNS(t *testing.T) {
 	if !tu.IsTravis() {
 		t.Parallel()

client/driver/docker_windows.go

@@ -11,3 +11,7 @@ const (
 func getPortBinding(ip string, port string) []docker.PortBinding {
 	return []docker.PortBinding{{HostIP: "", HostPort: port}}
 }
+
+func tweakCapabilities(basics, adds, drops []string) ([]string, error) {
+	return nil, nil
+}