consul/connect: Add support for Connect terminating gateways

This PR implements Nomad built-in support for running Consul Connect
terminating gateways. Such a gateway can be used by services running
inside the service mesh to access "legacy" services running outside
the service mesh while still making use of Consul's service identity
based networking and ACL policies.

https://www.consul.io/docs/connect/gateways/terminating-gateway

These gateways are declared as part of a task group level service
definition within the connect stanza.

service {
  connect {
    gateway {
      proxy {
        // envoy proxy configuration
      }
      terminating {
        // terminating-gateway configuration entry
      }
    }
  }
}

Currently Envoy is the only supported gateway implementation in
Consul. The gateay task can be customized by configuring the
connect.sidecar_task block.

When the gateway.terminating field is set, Nomad will write/update
the Configuration Entry into Consul on job submission. Because CEs
are global in scope and there may be more than one Nomad cluster
communicating with Consul, there is an assumption that any terminating
gateway defined in Nomad for a particular service will be the same
among Nomad clusters.

Gateways require Consul 1.8.0+, checked by a node constraint.

Closes #9445

Author: shoenig

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## 1.0.3 (Unreleased)
 
+FEATURES:
+
+* **Terminating Gateways**: Adds built-in support for running Consul Connect terminating gateways [[GH-9829](https://github.com/hashicorp/nomad/pull/9829)]
+
 BUG FIXES:
 
  * consul/connect: Fixed a bug where gateway proxy connection default timeout not set [[GH-9851](https://github.com/hashicorp/nomad/pull/9851)]

api/services.go

@@ -302,8 +302,8 @@ type ConsulGateway struct {
 	// Ingress represents the Consul Configuration Entry for an Ingress Gateway.
 	Ingress *ConsulIngressConfigEntry `hcl:"ingress,block"`
 
-	// Terminating is not yet supported.
-	// Terminating *ConsulTerminatingConfigEntry
+	// Terminating represents the Consul Configuration Entry for a Terminating Gateway.
+	Terminating *ConsulTerminatingConfigEntry `hcl:"terminating,block"`
 
 	// Mesh is not yet supported.
 	// Mesh *ConsulMeshConfigEntry
@@ -315,6 +315,7 @@ func (g *ConsulGateway) Canonicalize() {
 	}
 	g.Proxy.Canonicalize()
 	g.Ingress.Canonicalize()
+	g.Terminating.Canonicalize()
 }
 
 func (g *ConsulGateway) Copy() *ConsulGateway {
@@ -323,8 +324,9 @@ func (g *ConsulGateway) Copy() *ConsulGateway {
 	}
 
 	return &ConsulGateway{
-		Proxy:   g.Proxy.Copy(),
-		Ingress: g.Ingress.Copy(),
+		Proxy:       g.Proxy.Copy(),
+		Ingress:     g.Ingress.Copy(),
+		Terminating: g.Terminating.Copy(),
 	}
 }
 
@@ -335,8 +337,8 @@ type ConsulGatewayBindAddress struct {
 }
 
 var (
-	// defaultConnectTimeout is the default amount of time a connect gateway will
-	// wait for a response from an upstream service (same as consul)
+	// defaultGatewayConnectTimeout is the default amount of time connections to
+	// upstreams are allowed before timing out.
 	defaultGatewayConnectTimeout = 5 * time.Second
 )
 
@@ -349,6 +351,7 @@ type ConsulGatewayProxy struct {
 	EnvoyGatewayBindTaggedAddresses bool                                 `mapstructure:"envoy_gateway_bind_tagged_addresses" hcl:"envoy_gateway_bind_tagged_addresses,optional"`
 	EnvoyGatewayBindAddresses       map[string]*ConsulGatewayBindAddress `mapstructure:"envoy_gateway_bind_addresses" hcl:"envoy_gateway_bind_addresses,block"`
 	EnvoyGatewayNoDefaultBind       bool                                 `mapstructure:"envoy_gateway_no_default_bind" hcl:"envoy_gateway_no_default_bind,optional"`
+	EnvoyDNSDiscoveryType           string                               `mapstructure:"envoy_dns_discovery_type" hcl:"envoy_dns_discovery_type,optional"`
 	Config                          map[string]interface{}               `hcl:"config,block"` // escape hatch envoy config
 }
 
@@ -397,6 +400,7 @@ func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy {
 		EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses,
 		EnvoyGatewayBindAddresses:       binds,
 		EnvoyGatewayNoDefaultBind:       p.EnvoyGatewayNoDefaultBind,
+		EnvoyDNSDiscoveryType:           p.EnvoyDNSDiscoveryType,
 		Config:                          config,
 	}
 }
@@ -549,9 +553,74 @@ func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry {
 	}
 }
 
-// ConsulTerminatingConfigEntry is not yet supported.
-// type ConsulTerminatingConfigEntry struct {
-// }
+type ConsulLinkedService struct {
+	Name     string `hcl:"name,optional"`
+	CAFile   string `hcl:"ca_file,optional"`
+	CertFile string `hcl:"cert_file,optional"`
+	KeyFile  string `hcl:"key_file,optional"`
+	SNI      string `hcl:"sni,optional"`
+}
+
+func (s *ConsulLinkedService) Canonicalize() {
+	// nothing to do for now
+}
+
+func (s *ConsulLinkedService) Copy() *ConsulLinkedService {
+	if s == nil {
+		return nil
+	}
+
+	return &ConsulLinkedService{
+		Name:     s.Name,
+		CAFile:   s.CAFile,
+		CertFile: s.CertFile,
+		KeyFile:  s.KeyFile,
+		SNI:      s.SNI,
+	}
+}
+
+// ConsulTerminatingConfigEntry represents the Consul Configuration Entry type
+// for a Terminating Gateway.
+//
+// https://www.consul.io/docs/agent/config-entries/terminating-gateway#available-fields
+type ConsulTerminatingConfigEntry struct {
+	// Namespace is not yet supported.
+	// Namespace string
+
+	Services []*ConsulLinkedService `hcl:"service,block"`
+}
+
+func (e *ConsulTerminatingConfigEntry) Canonicalize() {
+	if e == nil {
+		return
+	}
+
+	if len(e.Services) == 0 {
+		e.Services = nil
+	}
+
+	for _, service := range e.Services {
+		service.Canonicalize()
+	}
+}
+
+func (e *ConsulTerminatingConfigEntry) Copy() *ConsulTerminatingConfigEntry {
+	if e == nil {
+		return nil
+	}
+
+	var services []*ConsulLinkedService = nil
+	if n := len(e.Services); n > 0 {
+		services = make([]*ConsulLinkedService, n)
+		for i := 0; i < n; i++ {
+			services[i] = e.Services[i].Copy()
+		}
+	}
+
+	return &ConsulTerminatingConfigEntry{
+		Services: services,
+	}
+}
 
 // ConsulMeshConfigEntry is not yet supported.
 // type ConsulMeshConfigEntry struct {

api/services_test.go

@@ -291,7 +291,10 @@ func TestService_ConsulGateway_Canonicalize(t *testing.T) {
 		}
 		cg.Canonicalize()
 		require.Equal(t, timeToPtr(5*time.Second), cg.Proxy.ConnectTimeout)
+		require.True(t, cg.Proxy.EnvoyGatewayBindTaggedAddresses)
 		require.Nil(t, cg.Proxy.EnvoyGatewayBindAddresses)
+		require.True(t, cg.Proxy.EnvoyGatewayNoDefaultBind)
+		require.Empty(t, cg.Proxy.EnvoyDNSDiscoveryType)
 		require.Nil(t, cg.Proxy.Config)
 		require.Nil(t, cg.Ingress.Listeners)
 	})
@@ -314,6 +317,7 @@ func TestService_ConsulGateway_Copy(t *testing.T) {
 				"listener2": {Address: "10.0.0.1", Port: 2001},
 			},
 			EnvoyGatewayNoDefaultBind: true,
+			EnvoyDNSDiscoveryType:     "STRICT_DNS",
 			Config: map[string]interface{}{
 				"foo": "bar",
 				"baz": 3,
@@ -334,6 +338,11 @@ func TestService_ConsulGateway_Copy(t *testing.T) {
 				}},
 			},
 		},
+		Terminating: &ConsulTerminatingConfigEntry{
+			Services: []*ConsulLinkedService{{
+				Name: "linked-service1",
+			}},
+		},
 	}
 
 	t.Run("complete", func(t *testing.T) {
@@ -418,3 +427,47 @@ func TestService_ConsulIngressConfigEntry_Copy(t *testing.T) {
 		require.Equal(t, entry, result)
 	})
 }
+
+func TestService_ConsulTerminatingConfigEntry_Canonicalize(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		c := (*ConsulTerminatingConfigEntry)(nil)
+		c.Canonicalize()
+		require.Nil(t, c)
+	})
+
+	t.Run("empty services", func(t *testing.T) {
+		c := &ConsulTerminatingConfigEntry{
+			Services: []*ConsulLinkedService{},
+		}
+		c.Canonicalize()
+		require.Nil(t, c.Services)
+	})
+}
+
+func TestService_ConsulTerminatingConfigEntry_Copy(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		result := (*ConsulIngressConfigEntry)(nil).Copy()
+		require.Nil(t, result)
+	})
+
+	entry := &ConsulTerminatingConfigEntry{
+		Services: []*ConsulLinkedService{{
+			Name: "servic1",
+		}, {
+			Name:     "service2",
+			CAFile:   "ca_file.pem",
+			CertFile: "cert_file.pem",
+			KeyFile:  "key_file.pem",
+			SNI:      "sni.terminating.consul",
+		}},
+	}
+
+	t.Run("complete", func(t *testing.T) {
+		result := entry.Copy()
+		require.Equal(t, entry, result)
+	})
+}

client/allocrunner/taskrunner/envoy_bootstrap_hook.go

@@ -110,13 +110,16 @@ func (envoyBootstrapHook) Name() string {
 	return envoyBootstrapHookName
 }
 
+func isConnectKind(kind string) bool {
+	kinds := []string{structs.ConnectProxyPrefix, structs.ConnectIngressPrefix, structs.ConnectTerminatingPrefix}
+	return helper.SliceStringContains(kinds, kind)
+}
+
 func (_ *envoyBootstrapHook) extractNameAndKind(kind structs.TaskKind) (string, string, error) {
-	serviceKind := kind.Name()
 	serviceName := kind.Value()
+	serviceKind := kind.Name()
 
-	switch serviceKind {
-	case structs.ConnectProxyPrefix, structs.ConnectIngressPrefix:
-	default:
+	if !isConnectKind(serviceKind) {
 		return "", "", errors.New("envoy must be used as connect sidecar or gateway")
 	}
 
@@ -350,13 +353,15 @@ func (h *envoyBootstrapHook) newEnvoyBootstrapArgs(
 		proxyID      string // gateway only
 	)
 
-	if service.Connect.HasSidecar() {
+	switch {
+	case service.Connect.HasSidecar():
 		sidecarForID = h.proxyServiceID(group, service)
-	}
-
-	if service.Connect.IsGateway() {
-		gateway = "ingress" // more types in the future
+	case service.Connect.IsIngress():
+		proxyID = h.proxyServiceID(group, service)
+		gateway = "ingress"
+	case service.Connect.IsTerminating():
 		proxyID = h.proxyServiceID(group, service)
+		gateway = "terminating"
 	}
 
 	h.logger.Debug("bootstrapping envoy",

command/agent/consul/connect.go

@@ -26,13 +26,15 @@ func newConnect(serviceName string, nc *structs.ConsulConnect, networks structs.
 		return &api.AgentServiceConnect{Native: true}, nil
 
 	case nc.HasSidecar():
+		// must register the sidecar for this service
 		sidecarReg, err := connectSidecarRegistration(serviceName, nc.SidecarService, networks)
 		if err != nil {
 			return nil, err
 		}
 		return &api.AgentServiceConnect{SidecarService: sidecarReg}, nil
 
 	default:
+		// a non-nil but empty connect block makes no sense
 		return nil, fmt.Errorf("Connect configuration empty for service %s", serviceName)
 	}
 }
@@ -64,6 +66,10 @@ func newConnectGateway(serviceName string, connect *structs.ConsulConnect) *api.
 			envoyConfig["envoy_gateway_bind_tagged_addresses"] = true
 		}
 
+		if proxy.EnvoyDNSDiscoveryType != "" {
+			envoyConfig["envoy_dns_discovery_type"] = proxy.EnvoyDNSDiscoveryType
+		}
+
 		if proxy.ConnectTimeout != nil {
 			envoyConfig["connect_timeout_ms"] = proxy.ConnectTimeout.Milliseconds()
 		}
@@ -89,7 +95,7 @@ func connectSidecarRegistration(serviceName string, css *structs.ConsulSidecarSe
 		return nil, err
 	}
 
-	proxy, err := connectProxy(css.Proxy, cPort.To, networks)
+	proxy, err := connectSidecarProxy(css.Proxy, cPort.To, networks)
 	if err != nil {
 		return nil, err
 	}
@@ -102,7 +108,7 @@ func connectSidecarRegistration(serviceName string, css *structs.ConsulSidecarSe
 	}, nil
 }
 
-func connectProxy(proxy *structs.ConsulProxy, cPort int, networks structs.Networks) (*api.AgentServiceConnectProxyConfig, error) {
+func connectSidecarProxy(proxy *structs.ConsulProxy, cPort int, networks structs.Networks) (*api.AgentServiceConnectProxyConfig, error) {
 	if proxy == nil {
 		proxy = new(structs.ConsulProxy)
 	}

command/agent/consul/connect_test.go

@@ -119,7 +119,7 @@ func TestConnect_connectProxy(t *testing.T) {
 	// If the input proxy is nil, we expect the output to be a proxy with its
 	// config set to default values.
 	t.Run("nil proxy", func(t *testing.T) {
-		proxy, err := connectProxy(nil, 2000, testConnectNetwork)
+		proxy, err := connectSidecarProxy(nil, 2000, testConnectNetwork)
 		require.NoError(t, err)
 		require.Equal(t, &api.AgentServiceConnectProxyConfig{
 			LocalServiceAddress: "",
@@ -134,7 +134,7 @@ func TestConnect_connectProxy(t *testing.T) {
 	})
 
 	t.Run("bad proxy", func(t *testing.T) {
-		_, err := connectProxy(&structs.ConsulProxy{
+		_, err := connectSidecarProxy(&structs.ConsulProxy{
 			LocalServiceAddress: "0.0.0.0",
 			LocalServicePort:    2000,
 			Upstreams:           nil,
@@ -149,7 +149,7 @@ func TestConnect_connectProxy(t *testing.T) {
 	})
 
 	t.Run("normal", func(t *testing.T) {
-		proxy, err := connectProxy(&structs.ConsulProxy{
+		proxy, err := connectSidecarProxy(&structs.ConsulProxy{
 			LocalServiceAddress: "0.0.0.0",
 			LocalServicePort:    2000,
 			Upstreams:           nil,
@@ -453,6 +453,7 @@ func TestConnect_newConnectGateway(t *testing.T) {
 						},
 					},
 					EnvoyGatewayNoDefaultBind: true,
+					EnvoyDNSDiscoveryType:     "STRICT_DNS",
 					Config: map[string]interface{}{
 						"foo": 1,
 					},
@@ -470,6 +471,7 @@ func TestConnect_newConnectGateway(t *testing.T) {
 					},
 				},
 				"envoy_gateway_no_default_bind": true,
+				"envoy_dns_discovery_type":      "STRICT_DNS",
 				"foo":                           1,
 			},
 		}, result)

command/agent/consul/service_client.go

@@ -891,10 +891,21 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w
 	// This enables the consul UI to show that Nomad registered this service
 	meta["external-source"] = "nomad"
 
-	// Explicitly set the service kind in case this service represents a Connect gateway.
+	// Explicitly set the Consul service Kind in case this service represents
+	// one of the Connect gateway types.
 	kind := api.ServiceKindTypical
-	if service.Connect.IsGateway() {
+	switch {
+	case service.Connect.IsIngress():
 		kind = api.ServiceKindIngressGateway
+	case service.Connect.IsTerminating():
+		kind = api.ServiceKindTerminatingGateway
+		// set the default port if bridge / default listener set
+		if defaultBind, exists := service.Connect.Gateway.Proxy.EnvoyGatewayBindAddresses["default"]; exists {
+			portLabel := fmt.Sprintf("%s-%s", structs.ConnectTerminatingPrefix, service.Name)
+			if dynPort, ok := workload.Ports.Get(portLabel); ok {
+				defaultBind.Port = dynPort.Value
+			}
+		}
 	}
 
 	// Build the Consul Service registration request