cni: add DNS set by CNI plugins to task configuration

CNI plugins may set DNS configuration, but this isn't threaded through to the
task configuration so that we can write it to the `/etc/resolv.conf` file as
needed. Add the `AllocNetworkStatus` to the alloc hook resources so they're
accessible from the taskrunner, which will prepend the DNS entries to any
entries provided by the user.

Fixes: #11102

.changelog/20007.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+cni: Fixed a bug where DNS set by CNI plugins was not provided to task drivers
+```

client/allocrunner/alloc_runner.go

@@ -909,6 +909,7 @@ func (ar *allocRunner) SetNetworkStatus(s *structs.AllocNetworkStatus) {
 	ar.stateLock.Lock()
 	defer ar.stateLock.Unlock()
 	ar.state.NetworkStatus = s.Copy()
+	ar.hookResources.SetAllocNetworkStatus(s) // will copy in getter
 }
 
 func (ar *allocRunner) NetworkStatus() *structs.AllocNetworkStatus {

client/allocrunner/taskrunner/task_runner.go

@@ -1135,6 +1135,22 @@ func (tr *TaskRunner) buildTaskConfig() *drivers.TaskConfig {
 				Options:  interpolatedNetworks[0].DNS.Options,
 			}
 		}
+
+		// merge in the DNS set by any CNI plugins
+		netStatus := tr.allocHookResources.GetAllocNetworkStatus()
+		if netStatus != nil && netStatus.DNS != nil {
+			if dns == nil {
+				dns = &drivers.DNSConfig{
+					Servers:  netStatus.DNS.Servers,
+					Searches: netStatus.DNS.Searches,
+					Options:  netStatus.DNS.Options,
+				}
+			} else {
+				dns.Servers = append(netStatus.DNS.Servers, dns.Servers...)
+				dns.Searches = append(netStatus.DNS.Searches, dns.Searches...)
+				dns.Options = append(netStatus.DNS.Options, dns.Options...)
+			}
+		}
 	}
 
 	memoryLimit := taskResources.Memory.MemoryMB

client/allocrunner/taskrunner/task_runner_test.go

@@ -2931,3 +2931,92 @@ func TestTaskRunner_IdentityHook_Disabled(t *testing.T) {
 	taskEnv := tr.envBuilder.Build()
 	must.MapNotContainsKey(t, taskEnv.EnvMap, "NOMAD_TOKEN")
 }
+
+func TestTaskRunner_AllocNetworkStatus(t *testing.T) {
+	ci.Parallel(t)
+
+	// Mock task with group network
+	alloc1 := mock.Alloc()
+	task1 := alloc1.Job.TaskGroups[0].Tasks[0]
+	alloc1.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
+		{
+			Device: "eth0",
+			IP:     "192.168.0.100",
+			DNS: &structs.DNSConfig{
+				Servers:  []string{"1.1.1.1", "8.8.8.8"},
+				Searches: []string{"test.local"},
+				Options:  []string{},
+			},
+			ReservedPorts: []structs.Port{{Label: "admin", Value: 5000}},
+			DynamicPorts:  []structs.Port{{Label: "http", Value: 9876}},
+		}}
+	task1.Driver = "mock_driver"
+	task1.Config = map[string]interface{}{"run_for": "2s"}
+
+	// Mock task with task networking only
+	alloc2 := mock.Alloc()
+	task2 := alloc2.Job.TaskGroups[0].Tasks[0]
+	task2.Driver = "mock_driver"
+	task2.Config = map[string]interface{}{"run_for": "2s"}
+
+	testCases := []struct {
+		name   string
+		alloc  *structs.Allocation
+		task   *structs.Task
+		expect *drivers.DNSConfig
+	}{
+		{
+			name:  "task with group networking",
+			alloc: alloc1,
+			task:  task1,
+			expect: &drivers.DNSConfig{
+				Servers:  []string{"10.37.105.17", "1.1.1.1", "8.8.8.8"},
+				Searches: []string{"node.consul", "test.local"},
+				Options:  []string{"ndots:2", "edns0"},
+			},
+		},
+		{
+			name:   "task without group networking",
+			alloc:  alloc2,
+			task:   task2,
+			expect: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+
+			conf, cleanup := testTaskRunnerConfig(t, tc.alloc, tc.task.Name, nil)
+			t.Cleanup(cleanup)
+
+			// note this will never actually be set if we don't have group/CNI
+			// networking, but it's a good validation no-group/CNI code path
+			conf.AllocHookResources.SetAllocNetworkStatus(&structs.AllocNetworkStatus{
+				InterfaceName: "",
+				Address:       "",
+				DNS: &structs.DNSConfig{
+					Servers:  []string{"10.37.105.17"},
+					Searches: []string{"node.consul"},
+					Options:  []string{"ndots:2", "edns0"},
+				},
+			})
+
+			tr, err := NewTaskRunner(conf)
+			must.NoError(t, err)
+
+			// Run the task runner.
+			go tr.Run()
+			t.Cleanup(func() {
+				tr.Kill(context.Background(), structs.NewTaskEvent("cleanup"))
+			})
+
+			// Wait for task to complete.
+			testWaitForTaskToStart(t, tr)
+
+			tr.stateLock.RLock()
+			t.Cleanup(tr.stateLock.RUnlock)
+
+			must.Eq(t, tc.expect, tr.localState.TaskHandle.Config.DNS)
+		})
+	}
+}

client/structs/allochook.go

@@ -9,15 +9,17 @@ import (
 	consulapi "github.com/hashicorp/consul/api"
 	"github.com/hashicorp/nomad/client/pluginmanager/csimanager"
 	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 // AllocHookResources contains data that is provided by AllocRunner Hooks for
 // consumption by TaskRunners. This should be instantiated once in the
 // AllocRunner and then only accessed via getters and setters that hold the
 // lock.
 type AllocHookResources struct {
-	csiMounts    map[string]*csimanager.MountInfo
-	consulTokens map[string]map[string]*consulapi.ACLToken // Consul cluster -> service identity -> token
+	csiMounts     map[string]*csimanager.MountInfo
+	consulTokens  map[string]map[string]*consulapi.ACLToken // Consul cluster -> service identity -> token
+	networkStatus *structs.AllocNetworkStatus
 
 	mu sync.RWMutex
 }
@@ -67,3 +69,21 @@ func (a *AllocHookResources) SetConsulTokens(m map[string]map[string]*consulapi.
 		a.consulTokens[k] = v
 	}
 }
+
+// GetAllocNetworkStatus returns a copy of the AllocNetworkStatus previously
+// written the group's network_hook
+func (a *AllocHookResources) GetAllocNetworkStatus() *structs.AllocNetworkStatus {
+	a.mu.RLock()
+	defer a.mu.RUnlock()
+
+	return a.networkStatus.Copy()
+}
+
+// SetAllocNetworkStatus stores the AllocNetworkStatus for later use by the
+// taskrunner's buildTaskConfig() method
+func (a *AllocHookResources) SetAllocNetworkStatus(ans *structs.AllocNetworkStatus) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	a.networkStatus = ans
+}