consul/connect: enable configuring custom gateway task

shoenig · shoenig · commit dd6bca092cf5 · 2020-12-15T09:40:38.000-06:00
Add the ability to configure the Task used for Connect gateways, similar to how sidecar Task can be configured. The implementation here simply re-uses the sidecar_task stanza, and now gets applied whether connect.sidecar_service or connect.gateway is the thing being defined. In retrospect, connect.sidecar_task could have been more generically named like connect.task to make it a little more re-usable. Closes #9474
diff --git a/nomad/job_endpoint_hook_connect.go b/nomad/job_endpoint_hook_connect.go
@@ -263,11 +263,16 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error {
 
 			// inject the gateway task only if it does not yet already exist
 			if !hasGatewayTaskForService(g, service.Name) {
-				// use the default envoy image, for now there is no support for a custom task
 				task := newConnectGatewayTask(service.Name, netHost)
 
 				g.Tasks = append(g.Tasks, task)
 
+				// the connect.sidecar_task stanza can also be used to configure
+				// a custom task to use as a gateway proxy
+				if service.Connect.SidecarTask != nil {
+					service.Connect.SidecarTask.MergeIntoTask(task)
+				}
+
 				task.Canonicalize(job, g)
 			}
 		}
diff --git a/nomad/job_endpoint_hook_connect_test.go b/nomad/job_endpoint_hook_connect_test.go
@@ -92,7 +92,7 @@ func TestJobEndpointConnect_groupConnectHook(t *testing.T) {
 	tgExp.Services[0].Name = "backend"
 	tgExp.Services[1].Name = "admin"
 
-	// Expect sidecar tasks to be properly canonicalized
+	// Expect sidecar tasks to be in canonical form.
 	tgExp.Tasks[0].Canonicalize(job, tgExp)
 	tgExp.Tasks[1].Canonicalize(job, tgExp)
 	tgExp.Networks[0].DynamicPorts = []structs.Port{{
@@ -146,6 +146,75 @@ func TestJobEndpointConnect_groupConnectHook_IngressGateway(t *testing.T) {
 	require.Exactly(t, expTG, job.TaskGroups[0])
 }
 
+func TestJobEndpointConnect_groupConnectHook_IngressGateway_CustomTask(t *testing.T) {
+	t.Parallel()
+
+	// Test that the connect gateway task is inserted if a gateway service exists
+	// and since this is a bridge network, will rewrite the default gateway proxy
+	// block with correct configuration.
+	job := mock.ConnectIngressGatewayJob("bridge", false)
+
+	job.Meta = map[string]string{
+		"gateway_name": "my-gateway",
+	}
+
+	job.TaskGroups[0].Services[0].Name = "${NOMAD_META_gateway_name}"
+	job.TaskGroups[0].Services[0].Connect.SidecarTask = &structs.SidecarTask{
+		Driver: "raw_exec",
+		User:   "sidecars",
+		Config: map[string]interface{}{
+			"command": "/bin/sidecar",
+			"args":    []string{"a", "b"},
+		},
+		Resources: &structs.Resources{
+			CPU: 400,
+			// Memory: inherit 128
+		},
+		KillSignal: "SIGHUP",
+	}
+
+	expTG := job.TaskGroups[0].Copy()
+	expTG.Tasks = []*structs.Task{
+		// inject merged gateway task
+		{
+			Name:   "connect-ingress-my-gateway",
+			Kind:   structs.NewTaskKind(structs.ConnectIngressPrefix, "my-gateway"),
+			Driver: "raw_exec",
+			User:   "sidecars",
+			Config: map[string]interface{}{
+				"command": "/bin/sidecar",
+				"args":    []string{"a", "b"},
+			},
+			Resources: &structs.Resources{
+				CPU:      400,
+				MemoryMB: 128,
+			},
+			LogConfig: &structs.LogConfig{
+				MaxFiles:      2,
+				MaxFileSizeMB: 2,
+			},
+			ShutdownDelay: 5 * time.Second,
+			KillSignal:    "SIGHUP",
+			Constraints: structs.Constraints{
+				connectGatewayVersionConstraint(),
+			},
+		},
+	}
+	expTG.Services[0].Name = "my-gateway"
+	expTG.Tasks[0].Canonicalize(job, expTG)
+	expTG.Networks[0].Canonicalize()
+
+	// rewrite the service gateway proxy configuration
+	expTG.Services[0].Connect.Gateway.Proxy = gatewayProxyForBridge(expTG.Services[0].Connect.Gateway)
+
+	require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
+	require.Exactly(t, expTG, job.TaskGroups[0])
+
+	// Test that the hook is idempotent
+	require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
+	require.Exactly(t, expTG, job.TaskGroups[0])
+}
+
 // TestJobEndpoint_ConnectInterpolation asserts that when a Connect sidecar
 // proxy task is being created for a group service with an interpolated name,
 // the service name is interpolated *before the task is created.
@@ -330,7 +399,7 @@ func TestJobEndpointConnect_gatewayProxyIsDefault(t *testing.T) {
 	t.Run("bind-addresses set", func(t *testing.T) {
 		result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{
 			EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{
-				"listener1": &structs.ConsulGatewayBindAddress{
+				"listener1": {
 					Address: "1.1.1.1",
 					Port:    9000,
 				},
@@ -362,7 +431,7 @@ func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) {
 			}},
 		})
 		require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{
-			"service1": &structs.ConsulGatewayBindAddress{
+			"service1": {
 				Address: "0.0.0.0",
 				Port:    3000,
 			},
@@ -388,15 +457,15 @@ func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) {
 			}},
 		})
 		require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{
-			"service1": &structs.ConsulGatewayBindAddress{
+			"service1": {
 				Address: "0.0.0.0",
 				Port:    3000,
 			},
-			"service2": &structs.ConsulGatewayBindAddress{
+			"service2": {
 				Address: "0.0.0.0",
 				Port:    3000,
 			},
-			"service3": &structs.ConsulGatewayBindAddress{
+			"service3": {
 				Address: "0.0.0.0",
 				Port:    3001,
 			},
diff --git a/website/pages/docs/job-specification/gateway.mdx b/website/pages/docs/job-specification/gateway.mdx
@@ -199,12 +199,30 @@ make use of the envoy version interpolation, e.g.
 meta.connect.gateway_image = custom/envoy-${NOMAD_envoy_version}:latest
 ```
 
+### Custom gateway task
+
+The task created for the gateway can be configured manually using the
+[`sidecar_task`][sidecar_task] stanza.
+
+```
+connect {
+  gateway {
+    # ...
+  }
+
+  sidecar_task {
+    # ...
+  }
+}
+```
+
 [proxy]: /docs/job-specification/gateway#proxy-parameters
 [ingress]: /docs/job-specification/gateway#ingress-parameters
 [tls]: /docs/job-specification/gateway#tls-parameters
 [listener]: /docs/job-specification/gateway#listener-parameters
 [service]: /docs/job-specification/gateway#service-parameters
 [service-default]: https://www.consul.io/docs/agent/config-entries/service-defaults
+[sidecar_task]: /docs/job-specification/sidecar_task
 [connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout
 [address]: /docs/job-specification/gateway#address-parameters
 [Advanced Configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration
diff --git a/website/pages/docs/job-specification/sidecar_task.mdx b/website/pages/docs/job-specification/sidecar_task.mdx
@@ -12,7 +12,7 @@ description: |-
 <Placement groups={['job', 'group', 'service', 'connect', 'sidecar_task']} />
 
 The `sidecar_task` stanza allows configuring various options for the proxy
-sidecar managed by Nomad for [Consul
+sidecar or connect gateway managed by Nomad for the [Consul
 Connect](/docs/integrations/consul-connect) integration such as
 resource requirements, kill timeouts and more as defined below. It is valid
 only within the context of a [`connect`][connect] stanza.
@@ -53,25 +53,30 @@ job "countdash" {
 }
 ```
 
-## Default Envoy proxy sidecar
+## Default Envoy configuration
 
-Nomad automatically includes a default Envoy proxy sidecar task whenever a
-group service has a [`sidecar_service`][sidecar_service] stanza.
+Nomad automatically launches and manages an Envoy task for use as a proxy sidecar
+or connect gateway, when [`sidecar_service`][sidecar_service] or [`gateway`][gateway]
+are configured.
 
-The default sidecar task is equivalent to:
+The default envoy task is equivalent to:
 
 ```hcl
 sidecar_task {
   name = "connect-proxy-<service>"
+  #      "connect-gateway-<service>" when used as a gateway
 
-  lifecycle {
+  lifecycle { # absent when used as a gateway
     hook    = "prestart"
     sidecar = true
   }
 
   driver = "docker"
+
   config {
     image = "${meta.connect.sidecar_image}"
+    #       "${meta.connect.gateway_image}" when used as a gateway
+
     args = [
       "-c",
       "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
@@ -97,13 +102,16 @@ sidecar_task {
 }
 ```
 
-The `meta.connect.sidecar_image`, `meta.connect.log_level`, and
-`meta.connect.proxy_concurrency` variables are [_client_
-configurable][nodemeta] variables with the following defaults:
+The `meta.connect.sidecar_image`, `meta.connect.gateway_image`, `meta.connect.log_level`,
+and `meta.connect.proxy_concurrency` variables are [client configurable][nodemeta]
+variables with the following defaults:
 
 - `sidecar_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official
    upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically
    by a query to Consul.
+- `gateway_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official
+   upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically
+   by a query to Consul.
 - `log_level` - `(string: "info")` - Envoy sidecar log level. "`debug`" is useful for
   debugging Connect related issues.
 - `proxy_concurrency` - `(string: "1")` - The number of [worker threads][worker_threads] the Envoy
@@ -118,8 +126,8 @@ meta.connect.sidecar_image = custom/envoy-${NOMAD_envoy_version}:latest
 
 ## `sidecar_task` Parameters
 
-- `name` `(string: "connect-proxy-<service>")` - Name of the task. Defaults to
-  including the name of the service it is a proxy for.
+- `name` `(string: "connect-[proxy|gateway]-<service>")` - Name of the task. Defaults to
+  including the name of the service the proxy or gateway is providing.
 
 - `driver` `(string: "docker")` - Driver used for the sidecar task.
 
@@ -166,12 +174,13 @@ The following example configures resources for the sidecar task and other config
 ```
 
 [connect]: /docs/job-specification/connect 'Nomad connect Job Specification'
-[job]: /docs/job-specification/job 'Nomad job Job Specification'
+[gateway]: /docs/job-specification/gateway
 [group]: /docs/job-specification/group 'Nomad group Job Specification'
-[task]: /docs/job-specification/task 'Nomad task Job Specification'
 [interpolation]: /docs/runtime/interpolation 'Nomad interpolation'
-[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification'
-[resources]: /docs/job-specification/resources 'Nomad resources Job Specification'
+[job]: /docs/job-specification/job 'Nomad job Job Specification'
 [logs]: /docs/job-specification/logs 'Nomad logs Job Specification'
+[resources]: /docs/job-specification/resources 'Nomad resources Job Specification'
+[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification'
+[task]: /docs/job-specification/task 'Nomad task Job Specification'
 [nodemeta]: /docs/configuration/client#meta
 [worker_threads]: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-concurrency
