Secret dir materialized in alloc/task directory

Author: dadgar

client/allocdir/alloc_dir.go

@@ -46,6 +46,10 @@ var (
 	// regardless of driver.
 	TaskLocal = "local"
 
+	// TaskSecrets is the the name of the secret directory inside each task
+	// directory
+	TaskSecrets = "secrets"
+
 	// TaskDirs is the set of directories created in each tasks directory.
 	TaskDirs = []string{"tmp"}
 )
@@ -154,6 +158,14 @@ func (d *AllocDir) UnmountAll() error {
 			}
 		}
 
+		taskSecret := filepath.Join(dir, TaskSecrets)
+		if d.pathExists(taskSecret) {
+			if err := d.removeSecretDir(taskSecret); err != nil {
+				mErr.Errors = append(mErr.Errors,
+					fmt.Errorf("failed to remove the secret dir %q: %v", taskSecret, err))
+			}
+		}
+
 		// Unmount dev/ and proc/ have been mounted.
 		d.unmountSpecialDirs(dir)
 	}
@@ -223,6 +235,16 @@ func (d *AllocDir) Build(tasks []*structs.Task) error {
 				return err
 			}
 		}
+
+		// Create the secret directory
+		secret := filepath.Join(taskDir, TaskSecrets)
+		if err := d.createSecretDir(secret); err != nil {
+			return err
+		}
+
+		if err := d.dropDirPermissions(secret); err != nil {
+			return err
+		}
 	}
 
 	return nil

client/allocdir/alloc_dir_darwin.go

@@ -1,6 +1,7 @@
 package allocdir
 
 import (
+	"os"
 	"syscall"
 )
 
@@ -14,6 +15,16 @@ func (d *AllocDir) unmountSharedDir(dir string) error {
 	return syscall.Unlink(dir)
 }
 
+// createSecretDir creates the secrets dir folder at the given path
+func (d *AllocDir) createSecretDir(dir string) error {
+	return os.MkdirAll(dir, 0777)
+}
+
+// removeSecretDir removes the secrets dir folder
+func (d *AllocDir) removeSecretDir(dir string) error {
+	return os.RemoveAll(dir)
+}
+
 // MountSpecialDirs mounts the dev and proc file system on the chroot of the
 // task. It's a no-op on darwin.
 func (d *AllocDir) MountSpecialDirs(taskDir string) error {

client/allocdir/alloc_dir_freebsd.go

@@ -1,6 +1,7 @@
 package allocdir
 
 import (
+	"os"
 	"syscall"
 )
 
@@ -14,6 +15,16 @@ func (d *AllocDir) unmountSharedDir(dir string) error {
 	return syscall.Unlink(dir)
 }
 
+// createSecretDir creates the secrets dir folder at the given path
+func (d *AllocDir) createSecretDir(dir string) error {
+	return os.MkdirAll(dir, 0777)
+}
+
+// removeSecretDir removes the secrets dir folder
+func (d *AllocDir) removeSecretDir(dir string) error {
+	return os.RemoveAll(dir)
+}
+
 // MountSpecialDirs mounts the dev and proc file system on the chroot of the
 // task. It's a no-op on FreeBSD right now.
 func (d *AllocDir) MountSpecialDirs(taskDir string) error {

client/allocdir/alloc_dir_linux.go

@@ -6,9 +6,16 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"golang.org/x/sys/unix"
+
 	"github.com/hashicorp/go-multierror"
 )
 
+const (
+	// secretDirTmpfsSize is the size of the tmpfs per task in MBs
+	secretDirTmpfsSize = 1
+)
+
 // Bind mounts the shared directory into the task directory. Must be root to
 // run.
 func (d *AllocDir) mountSharedDir(taskDir string) error {
@@ -23,6 +30,36 @@ func (d *AllocDir) unmountSharedDir(dir string) error {
 	return syscall.Unmount(dir, 0)
 }
 
+// createSecretDir creates the secrets dir folder at the given path using a
+// tmpfs
+func (d *AllocDir) createSecretDir(dir string) error {
+	// Only mount the tmpfs if we are root
+	if unix.Geteuid() == 0 {
+		if err := os.MkdirAll(dir, 0777); err != nil {
+			return err
+		}
+
+		var flags uintptr
+		flags = syscall.MS_NOEXEC
+		options := fmt.Sprintf("size=%dm", secretDirTmpfsSize)
+		err := syscall.Mount("tmpfs", dir, "tmpfs", flags, options)
+		return os.NewSyscallError("mount", err)
+	}
+
+	return os.MkdirAll(dir, 0777)
+}
+
+// createSecretDir removes the secrets dir folder
+func (d *AllocDir) removeSecretDir(dir string) error {
+	if unix.Geteuid() == 0 {
+		if err := syscall.Unmount(dir, 0); err != nil {
+			return err
+		}
+	}
+
+	return os.RemoveAll(dir)
+}
+
 // MountSpecialDirs mounts the dev and proc file system from the host to the
 // chroot
 func (d *AllocDir) MountSpecialDirs(taskDir string) error {

client/allocdir/alloc_dir_test.go

@@ -73,6 +73,10 @@ func TestAllocDir_BuildAlloc(t *testing.T) {
 		if _, err := os.Stat(tDir); os.IsNotExist(err) {
 			t.Fatalf("Build(%v) didn't create TaskDir %v", tasks, tDir)
 		}
+
+		if _, err := os.Stat(filepath.Join(tDir, TaskSecrets)); os.IsNotExist(err) {
+			t.Fatalf("Build(%v) didn't create secret dir %v", tasks)
+		}
 	}
 }
 

client/allocdir/alloc_dir_unix.go

@@ -14,11 +14,17 @@ import (
 )
 
 var (
-	//Path inside container for mounted directory shared across tasks in a task group.
+	// SharedAllocContainerPath is the path inside container for mounted
+	// directory shared across tasks in a task group.
 	SharedAllocContainerPath = filepath.Join("/", SharedAllocName)
 
-	//Path inside container for mounted directory for local storage.
+	// TaskLocalContainer is the path inside a container for mounted directory
+	// for local storage.
 	TaskLocalContainerPath = filepath.Join("/", TaskLocal)
+
+	// TaskSecretsContainerPath is the path inside a container for mounted
+	// secrets directory
+	TaskSecretsContainerPath = filepath.Join("/", TaskSecrets)
 )
 
 func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error {

client/allocdir/alloc_dir_windows.go

@@ -7,11 +7,17 @@ import (
 )
 
 var (
-	//Path inside container for mounted directory that is shared across tasks in a task group.
+	// SharedAllocContainerPath is the path inside container for mounted
+	// directory shared across tasks in a task group.
 	SharedAllocContainerPath = filepath.Join("c:\\", SharedAllocName)
 
-	//Path inside container for mounted directory for local storage.
+	// TaskLocalContainer is the path inside a container for mounted directory
+	// for local storage.
 	TaskLocalContainerPath = filepath.Join("c:\\", TaskLocal)
+
+	// TaskSecretsContainerPath is the path inside a container for mounted
+	// secrets directory
+	TaskSecretsContainerPath = filepath.Join("c:\\", TaskSecrets)
 )
 
 func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error {
@@ -23,6 +29,16 @@ func (d *AllocDir) mountSharedDir(dir string) error {
 	return errors.New("Mount on Windows not supported.")
 }
 
+// createSecretDir creates the secrets dir folder at the given path
+func (d *AllocDir) createSecretDir(dir string) error {
+	return os.MkdirAll(dir, 0777)
+}
+
+// removeSecretDir removes the secrets dir folder
+func (d *AllocDir) removeSecretDir(dir string) error {
+	return os.RemoveAll(dir)
+}
+
 // The windows version does nothing currently.
 func (d *AllocDir) dropDirPermissions(path string) error {
 	return nil

client/client_test.go

@@ -575,6 +575,14 @@ func TestClient_SaveRestoreState(t *testing.T) {
 	}, func(err error) {
 		t.Fatalf("err: %v", err)
 	})
+
+	// Destroy all the allocations
+	c2.allocLock.Lock()
+	for _, ar := range c2.allocs {
+		ar.Destroy()
+		<-ar.WaitCh()
+	}
+	c2.allocLock.Unlock()
 }
 
 func TestClient_Init(t *testing.T) {
@@ -608,6 +616,7 @@ func TestClient_BlockedAllocations(t *testing.T) {
 	c1 := testClient(t, func(c *config.Config) {
 		c.RPCHandler = s1
 	})
+	defer c1.Shutdown()
 
 	// Wait for the node to be ready
 	state := s1.State()
@@ -691,4 +700,13 @@ func TestClient_BlockedAllocations(t *testing.T) {
 	}, func(err error) {
 		t.Fatalf("err: %v", err)
 	})
+
+	// Destroy all the allocations
+	c1.allocLock.Lock()
+	for _, ar := range c1.allocs {
+		ar.Destroy()
+		<-ar.WaitCh()
+	}
+	c1.allocLock.Unlock()
+
 }

client/driver/docker.go

@@ -381,6 +381,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task,
 	// Set environment variables.
 	d.taskEnv.SetAllocDir(allocdir.SharedAllocContainerPath)
 	d.taskEnv.SetTaskLocalDir(allocdir.TaskLocalContainerPath)
+	d.taskEnv.SetTaskLocalDir(allocdir.TaskSecretsContainerPath)
 
 	config := &docker.Config{
 		Image:     driverConfig.ImageName,

client/driver/docker_test.go

@@ -117,7 +117,8 @@ func dockerSetup(t *testing.T, task *structs.Task) (*docker.Client, DriverHandle
 
 // This test should always pass, even if docker daemon is not available
 func TestDockerDriver_Fingerprint(t *testing.T) {
-	driverCtx, _ := testDriverContexts(&structs.Task{Name: "foo", Resources: basicResources})
+	driverCtx, execCtx := testDriverContexts(&structs.Task{Name: "foo", Resources: basicResources})
+	defer execCtx.AllocDir.Destroy()
 	d := NewDockerDriver(driverCtx)
 	node := &structs.Node{
 		Attributes: make(map[string]string),

client/driver/driver.go

@@ -153,6 +153,7 @@ func GetTaskEnv(allocDir *allocdir.AllocDir, node *structs.Node,
 		}
 
 		env.SetTaskLocalDir(filepath.Join(taskdir, allocdir.TaskLocal))
+		env.SetSecretDir(filepath.Join(taskdir, allocdir.TaskSecrets))
 	}
 
 	if task.Resources != nil {

client/driver/env/env.go

@@ -21,6 +21,10 @@ const (
 	// removed.
 	TaskLocalDir = "NOMAD_TASK_DIR"
 
+	// SecretDir is the environment variable with the path to the tasks secret
+	// directory where it can store sensitive data.
+	SecretDir = "NOMAD_SECRET_DIR"
+
 	// MemLimit is the environment variable with the tasks memory limit in MBs.
 	MemLimit = "NOMAD_MEMORY_LIMIT"
 
@@ -79,6 +83,7 @@ type TaskEnvironment struct {
 	JobMeta       map[string]string
 	AllocDir      string
 	TaskDir       string
+	SecretDir     string
 	CpuLimit      int
 	MemLimit      int
 	TaskName      string
@@ -153,6 +158,9 @@ func (t *TaskEnvironment) Build() *TaskEnvironment {
 	if t.TaskDir != "" {
 		t.TaskEnv[TaskLocalDir] = t.TaskDir
 	}
+	if t.SecretDir != "" {
+		t.TaskEnv[SecretDir] = t.SecretDir
+	}
 
 	// Build the resource limits
 	if t.MemLimit != 0 {
@@ -249,6 +257,16 @@ func (t *TaskEnvironment) ClearTaskLocalDir() *TaskEnvironment {
 	return t
 }
 
+func (t *TaskEnvironment) SetSecretDir(dir string) *TaskEnvironment {
+	t.SecretDir = dir
+	return t
+}
+
+func (t *TaskEnvironment) ClearSecretDir() *TaskEnvironment {
+	t.SecretDir = ""
+	return t
+}
+
 func (t *TaskEnvironment) SetMemLimit(limit int) *TaskEnvironment {
 	t.MemLimit = limit
 	return t

client/driver/exec_test.go

@@ -26,7 +26,8 @@ func TestExecDriver_Fingerprint(t *testing.T) {
 		Name:      "foo",
 		Resources: structs.DefaultResources(),
 	}
-	driverCtx, _ := testDriverContexts(task)
+	driverCtx, execCtx := testDriverContexts(task)
+	defer execCtx.AllocDir.Destroy()
 	d := NewExecDriver(driverCtx)
 	node := &structs.Node{
 		Attributes: map[string]string{

client/driver/executor/executor_linux_test.go

@@ -81,7 +81,23 @@ func TestExecutor_IsolationAndConstraints(t *testing.T) {
 		t.Fatalf("file %v hasn't been removed", memLimits)
 	}
 
-	expected := "/:\nalloc/\nbin/\ndev/\netc/\nlib/\nlib64/\nlocal/\nproc/\ntmp/\nusr/\n\n/etc/:\nld.so.cache\nld.so.conf\nld.so.conf.d/"
+	expected := `/:
+alloc/
+bin/
+dev/
+etc/
+lib/
+lib64/
+local/
+proc/
+secrets/
+tmp/
+usr/
+
+/etc/:
+ld.so.cache
+ld.so.conf
+ld.so.conf.d/`
 	file := filepath.Join(ctx.AllocDir.LogDir(), "web.stdout.0")
 	output, err := ioutil.ReadFile(file)
 	if err != nil {

client/driver/java_test.go

@@ -35,7 +35,8 @@ func TestJavaDriver_Fingerprint(t *testing.T) {
 		Name:      "foo",
 		Resources: structs.DefaultResources(),
 	}
-	driverCtx, _ := testDriverContexts(task)
+	driverCtx, execCtx := testDriverContexts(task)
+	defer execCtx.AllocDir.Destroy()
 	d := NewJavaDriver(driverCtx)
 	node := &structs.Node{
 		Attributes: map[string]string{

client/driver/qemu_test.go

@@ -19,7 +19,8 @@ func TestQemuDriver_Fingerprint(t *testing.T) {
 		Name:      "foo",
 		Resources: structs.DefaultResources(),
 	}
-	driverCtx, _ := testDriverContexts(task)
+	driverCtx, execCtx := testDriverContexts(task)
+	defer execCtx.AllocDir.Destroy()
 	d := NewQemuDriver(driverCtx)
 	node := &structs.Node{
 		Attributes: make(map[string]string),

client/driver/raw_exec_test.go

@@ -21,7 +21,8 @@ func TestRawExecDriver_Fingerprint(t *testing.T) {
 		Name:      "foo",
 		Resources: structs.DefaultResources(),
 	}
-	driverCtx, _ := testDriverContexts(task)
+	driverCtx, execCtx := testDriverContexts(task)
+	defer execCtx.AllocDir.Destroy()
 	d := NewRawExecDriver(driverCtx)
 	node := &structs.Node{
 		Attributes: make(map[string]string),

client/driver/rkt_test.go

@@ -376,6 +376,7 @@ func TestRktTaskValidate(t *testing.T) {
 			"dns_servers":        []string{"8.8.8.8", "8.8.4.4"},
 			"dns_search_domains": []string{"example.com", "example.org", "example.net"},
 		},
+		Resources: basicResources,
 	}
 	driverCtx, execCtx := testDriverContexts(task)
 	defer execCtx.AllocDir.Destroy()