cni: add DNS set by CNI plugins to task configuration

.changelog/20007.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+cni: Fixed a bug where DNS set by CNI plugins was not provided to task drivers
+```

client/allocrunner/alloc_runner.go

@@ -909,6 +909,7 @@ func (ar *allocRunner) SetNetworkStatus(s *structs.AllocNetworkStatus) {
 	ar.stateLock.Lock()
 	defer ar.stateLock.Unlock()
 	ar.state.NetworkStatus = s.Copy()
+	ar.hookResources.SetAllocNetworkStatus(s) // will copy in getter
 }
 
 func (ar *allocRunner) NetworkStatus() *structs.AllocNetworkStatus {

client/allocrunner/taskrunner/task_runner.go

@@ -1135,6 +1135,22 @@ func (tr *TaskRunner) buildTaskConfig() *drivers.TaskConfig {
 				Options:  interpolatedNetworks[0].DNS.Options,
 			}
 		}
+
+		// merge in the DNS set by any CNI plugins
+		netStatus := tr.allocHookResources.GetAllocNetworkStatus()
+		if netStatus != nil && netStatus.DNS != nil {
+			if dns == nil {
+				dns = &drivers.DNSConfig{
+					Servers:  netStatus.DNS.Servers,
+					Searches: netStatus.DNS.Searches,
+					Options:  netStatus.DNS.Options,
+				}
+			} else {
+				dns.Servers = append(netStatus.DNS.Servers, dns.Servers...)
+				dns.Searches = append(netStatus.DNS.Searches, dns.Searches...)
+				dns.Options = append(netStatus.DNS.Options, dns.Options...)
+			}
+		}
 	}
 
 	memoryLimit := taskResources.Memory.MemoryMB

client/allocrunner/taskrunner/task_runner_test.go

@@ -2931,3 +2931,92 @@ func TestTaskRunner_IdentityHook_Disabled(t *testing.T) {
 	taskEnv := tr.envBuilder.Build()
 	must.MapNotContainsKey(t, taskEnv.EnvMap, "NOMAD_TOKEN")
 }
+
+func TestTaskRunner_AllocNetworkStatus(t *testing.T) {
+	ci.Parallel(t)
+
+	// Mock task with group network
+	alloc1 := mock.Alloc()
+	task1 := alloc1.Job.TaskGroups[0].Tasks[0]
+	alloc1.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
+		{
+			Device: "eth0",
+			IP:     "192.168.0.100",
+			DNS: &structs.DNSConfig{
+				Servers:  []string{"1.1.1.1", "8.8.8.8"},
+				Searches: []string{"test.local"},
+				Options:  []string{},
+			},
+			ReservedPorts: []structs.Port{{Label: "admin", Value: 5000}},
+			DynamicPorts:  []structs.Port{{Label: "http", Value: 9876}},
+		}}
+	task1.Driver = "mock_driver"
+	task1.Config = map[string]interface{}{"run_for": "2s"}
+
+	// Mock task with task networking only
+	alloc2 := mock.Alloc()
+	task2 := alloc2.Job.TaskGroups[0].Tasks[0]
+	task2.Driver = "mock_driver"
+	task2.Config = map[string]interface{}{"run_for": "2s"}
+
+	testCases := []struct {
+		name   string
+		alloc  *structs.Allocation
+		task   *structs.Task
+		expect *drivers.DNSConfig
+	}{
+		{
+			name:  "task with group networking",
+			alloc: alloc1,
+			task:  task1,
+			expect: &drivers.DNSConfig{
+				Servers:  []string{"10.37.105.17", "1.1.1.1", "8.8.8.8"},
+				Searches: []string{"node.consul", "test.local"},
+				Options:  []string{"ndots:2", "edns0"},
+			},
+		},
+		{
+			name:   "task without group networking",
+			alloc:  alloc2,
+			task:   task2,
+			expect: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+
+			conf, cleanup := testTaskRunnerConfig(t, tc.alloc, tc.task.Name, nil)
+			t.Cleanup(cleanup)
+
+			// note this will never actually be set if we don't have group/CNI
+			// networking, but it's a good validation no-group/CNI code path
+			conf.AllocHookResources.SetAllocNetworkStatus(&structs.AllocNetworkStatus{
+				InterfaceName: "",
+				Address:       "",
+				DNS: &structs.DNSConfig{
+					Servers:  []string{"10.37.105.17"},
+					Searches: []string{"node.consul"},
+					Options:  []string{"ndots:2", "edns0"},
+				},
+			})
+
+			tr, err := NewTaskRunner(conf)
+			must.NoError(t, err)
+
+			// Run the task runner.
+			go tr.Run()
+			t.Cleanup(func() {
+				tr.Kill(context.Background(), structs.NewTaskEvent("cleanup"))
+			})
+
+			// Wait for task to complete.
+			testWaitForTaskToStart(t, tr)
+
+			tr.stateLock.RLock()
+			t.Cleanup(tr.stateLock.RUnlock)
+
+			must.Eq(t, tc.expect, tr.localState.TaskHandle.Config.DNS)
+		})
+	}
+}

client/structs/allochook.go

@@ -9,15 +9,17 @@ import (
 	consulapi "github.com/hashicorp/consul/api"
 	"github.com/hashicorp/nomad/client/pluginmanager/csimanager"
 	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 // AllocHookResources contains data that is provided by AllocRunner Hooks for
 // consumption by TaskRunners. This should be instantiated once in the
 // AllocRunner and then only accessed via getters and setters that hold the
 // lock.
 type AllocHookResources struct {
-	csiMounts    map[string]*csimanager.MountInfo
-	consulTokens map[string]map[string]*consulapi.ACLToken // Consul cluster -> service identity -> token
+	csiMounts     map[string]*csimanager.MountInfo
+	consulTokens  map[string]map[string]*consulapi.ACLToken // Consul cluster -> service identity -> token
+	networkStatus *structs.AllocNetworkStatus
 
 	mu sync.RWMutex
 }
@@ -67,3 +69,21 @@ func (a *AllocHookResources) SetConsulTokens(m map[string]map[string]*consulapi.
 		a.consulTokens[k] = v
 	}
 }
+
+// GetAllocNetworkStatus returns a copy of the AllocNetworkStatus previously
+// written the group's network_hook
+func (a *AllocHookResources) GetAllocNetworkStatus() *structs.AllocNetworkStatus {
+	a.mu.RLock()
+	defer a.mu.RUnlock()
+
+	return a.networkStatus.Copy()
+}
+
+// SetAllocNetworkStatus stores the AllocNetworkStatus for later use by the
+// taskrunner's buildTaskConfig() method
+func (a *AllocHookResources) SetAllocNetworkStatus(ans *structs.AllocNetworkStatus) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	a.networkStatus = ans
+}