consul/connect: add support for running connect native tasks

This PR adds the capability of running Connect Native Tasks on Nomad,
particularly when TLS and ACLs are enabled on Consul.

The `connect` stanza now includes a `native` parameter, which can be
set to the name of task that backs the Connect Native Consul service.

There is a new Client configuration parameter for the `consul` stanza
called `share_ssl`. Like `allow_unauthenticated` the default value is
true, but recommended to be disabled in production environments. When
enabled, the Nomad Client's Consul TLS information is shared with
Connect Native tasks through the normal Consul environment variables.
This does NOT include auth or token information.

If Consul ACLs are enabled, Service Identity Tokens are automatically
and injected into the Connect Native task through the CONSUL_HTTP_TOKEN
environment variable.

Any of the automatically set environment variables can be overridden by
the Connect Native task using the `env` stanza.

Fixes #6083

Author: shoenig

api/services.go

@@ -140,7 +140,7 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) {
 
 // ConsulConnect represents a Consul Connect jobspec stanza.
 type ConsulConnect struct {
-	Native         bool
+	Native         string
 	SidecarService *ConsulSidecarService `mapstructure:"sidecar_service"`
 	SidecarTask    *SidecarTask          `mapstructure:"sidecar_task"`
 }

api/services_test.go

@@ -69,7 +69,7 @@ func TestService_Connect_Canonicalize(t *testing.T) {
 	t.Run("empty connect", func(t *testing.T) {
 		cc := new(ConsulConnect)
 		cc.Canonicalize()
-		require.False(t, cc.Native)
+		require.Empty(t, cc.Native)
 		require.Nil(t, cc.SidecarService)
 		require.Nil(t, cc.SidecarTask)
 	})

client/allocrunner/consulsock_hook.go

@@ -52,7 +52,7 @@ func (*consulSockHook) Name() string {
 func (h *consulSockHook) shouldRun() bool {
 	tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup)
 	for _, s := range tg.Services {
-		if s.Connect != nil {
+		if s.Connect.HasSidecar() {
 			return true
 		}
 	}

client/allocrunner/taskrunner/connect_native_hook.go

@@ -0,0 +1,215 @@
+package taskrunner
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	hclog "github.com/hashicorp/go-hclog"
+	ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/nomad/structs/config"
+	"github.com/pkg/errors"
+)
+
+const (
+	connectNativeHookName = "connect_native"
+)
+
+type connectNativeHookConfig struct {
+	consulShareTLS bool
+	consul         consulTransportConfig
+	alloc          *structs.Allocation
+	logger         hclog.Logger
+}
+
+func newConnectNativeHookConfig(alloc *structs.Allocation, consul *config.ConsulConfig, logger hclog.Logger) *connectNativeHookConfig {
+	return &connectNativeHookConfig{
+		alloc:          alloc,
+		logger:         logger,
+		consulShareTLS: consul.ShareSSL == nil || *consul.ShareSSL, // default enabled
+		consul:         newConsulTransportConfig(consul),
+	}
+}
+
+// connectNativeHook manages additional automagic configuration for a connect
+// native task.
+//
+// If nomad client is configured to talk to Consul using TLS (or other special
+// auth), the native task will inherit that configuration EXCEPT for the consul
+// token.
+//
+// If consul is configured with ACLs enabled, a Service Identity token will be
+// generated on behalf of the native service and supplied to the task.
+type connectNativeHook struct {
+	// alloc is the allocation with the connect native task being run
+	alloc *structs.Allocation
+
+	// consulShareTLS is used to toggle whether the TLS configuration of the
+	// Nomad Client may be shared with Connect Native applications.
+	consulShareTLS bool
+
+	// consulConfig is used to enable the connect native enabled task to
+	// communicate with consul directly, as is necessary for the task to request
+	// its connect mTLS certificates.
+	consulConfig consulTransportConfig
+
+	// logger is used to log things
+	logger hclog.Logger
+}
+
+func newConnectNativeHook(c *connectNativeHookConfig) *connectNativeHook {
+	return &connectNativeHook{
+		alloc:          c.alloc,
+		consulShareTLS: c.consulShareTLS,
+		consulConfig:   c.consul,
+		logger:         c.logger.Named(connectNativeHookName),
+	}
+}
+
+func (connectNativeHook) Name() string {
+	return connectNativeHookName
+}
+
+func (h *connectNativeHook) Prestart(
+	ctx context.Context,
+	request *ifs.TaskPrestartRequest,
+	response *ifs.TaskPrestartResponse) error {
+
+	if !request.Task.Kind.IsConnectNative() {
+		response.Done = true
+		return nil
+	}
+
+	h.logger.Debug("share consul TLS configuration for connect native", "enabled", h.consulShareTLS, "task", request.Task.Name)
+	if h.consulShareTLS {
+		// copy TLS certificates
+		if err := h.copyCertificates(h.consulConfig, request.TaskDir.SecretsDir); err != nil {
+			h.logger.Error("failed to copy Consul TLS certificates", "error", err)
+			return err
+		}
+
+		// set environment variables for communicating with Consul agent, but
+		// only if those environment variables are not already set
+		response.Env = h.tlsEnv(request.TaskEnv.EnvMap)
+
+	}
+
+	if err := h.maybeSetSITokenEnv(request.TaskDir.SecretsDir, request.Task.Name, response.Env); err != nil {
+		h.logger.Error("failed to load Consul Service Identity Token", "error", err, "task", request.Task.Name)
+		return err
+	}
+
+	// tls/acl setup for native task done
+	response.Done = true
+	return nil
+}
+
+const (
+	secretCAFilename       = "consul_ca_file"
+	secretCertfileFilename = "consul_cert_file"
+	secretKeyfileFilename  = "consul_key_file"
+)
+
+func (h *connectNativeHook) copyCertificates(consulConfig consulTransportConfig, dir string) error {
+	if err := h.copyCertificate(consulConfig.CAFile, dir, secretCAFilename); err != nil {
+		return err
+	}
+	if err := h.copyCertificate(consulConfig.CertFile, dir, secretCertfileFilename); err != nil {
+		return err
+	}
+	if err := h.copyCertificate(consulConfig.KeyFile, dir, secretKeyfileFilename); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (connectNativeHook) copyCertificate(source, dir, name string) error {
+	if source == "" {
+		return nil
+	}
+
+	original, err := os.Open(source)
+	if err != nil {
+		return errors.Wrap(err, "failed to open consul TLS certificate")
+	}
+	defer original.Close()
+
+	destination := filepath.Join(dir, name)
+	fd, err := os.Create(destination)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create secrets/%s", name)
+	}
+	defer fd.Close()
+
+	if _, err := io.Copy(fd, original); err != nil {
+		return errors.Wrapf(err, "failed to copy certificate secrets/%s", name)
+	}
+
+	if err := fd.Sync(); err != nil {
+		return errors.Wrapf(err, "failed to write secrets/%s", name)
+	}
+
+	return nil
+}
+
+// tlsEnv creates a set of additional of environment variables to be used when launching
+// the connect native task. This will enable the task to communicate with Consul
+// if Consul has transport security turned on.
+//
+// We do NOT set CONSUL_HTTP_TOKEN from the nomad agent's consul config, as that
+// is a separate security concern addressed by the service identity hook.
+func (h *connectNativeHook) tlsEnv(env map[string]string) map[string]string {
+	m := make(map[string]string)
+
+	if _, exists := env["CONSUL_CACERT"]; !exists && h.consulConfig.CAFile != "" {
+		m["CONSUL_CACERT"] = filepath.Join("/secrets", secretCAFilename)
+	}
+
+	if _, exists := env["CONSUL_CLIENT_CERT"]; !exists && h.consulConfig.CertFile != "" {
+		m["CONSUL_CLIENT_CERT"] = filepath.Join("/secrets", secretCertfileFilename)
+	}
+
+	if _, exists := env["CONSUL_CLIENT_KEY"]; !exists && h.consulConfig.KeyFile != "" {
+		m["CONSUL_CLIENT_KEY"] = filepath.Join("/secrets", secretKeyfileFilename)
+	}
+
+	if _, exists := env["CONSUL_HTTP_SSL"]; !exists {
+		if v := h.consulConfig.SSL; v != "" {
+			m["CONSUL_HTTP_SSL"] = v
+		}
+	}
+
+	if _, exists := env["CONSUL_HTTP_SSL_VERIFY"]; !exists {
+		if v := h.consulConfig.VerifySSL; v != "" {
+			m["CONSUL_HTTP_SSL_VERIFY"] = v
+		}
+	}
+
+	return m
+}
+
+// maybeSetSITokenEnv will set the CONSUL_HTTP_TOKEN environment variable in
+// the given env map, if the token is found to exist in the task's secrets
+// directory.
+//
+// Following the pattern of the envoy_bootstrap_hook, the Consul Service Identity
+// ACL Token is generated prior to this hook, if Consul ACLs are enabled. This is
+// done in the sids_hook, which places the token at secrets/si_token in the task
+// workspace. The content of that file is the SI token specific to this task
+// instance.
+func (h *connectNativeHook) maybeSetSITokenEnv(dir, task string, env map[string]string) error {
+	token, err := ioutil.ReadFile(filepath.Join(dir, sidsTokenFile))
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return errors.Wrapf(err, "failed to load SI token for native task %s", task)
+		}
+		h.logger.Trace("no SI token to load for native task", "task", task)
+		return nil // token file DNE; acls not enabled
+	}
+	h.logger.Trace("recovered pre-existing SI token for native task", "task", task)
+	env["CONSUL_HTTP_TOKEN"] = string(token)
+	return nil
+}