WIP: refactoring to use multus pkg funcs

Signed-off-by: Miguel Duarte Barroso <[email protected]>

pkg/cniclient/types.go

@@ -3,6 +3,7 @@ package cniclient
 import (
 	"context"
 	"fmt"
+	"github.com/containernetworking/cni/pkg/skel"
 	"os"
 	"sort"
 	"strings"
@@ -21,13 +22,10 @@ const hostProcessPID = 1
 
 // CNIParams represent
 type CNIParams struct {
+	CniCmdArgs  skel.CmdArgs
 	Namespace   string
 	PodName     string
-	SandboxID   string
-	NetnsPath   string
 	NetworkName string
-	IfaceName   string
-	IfMAC       string
 }
 
 // CniPlugin represents a CNI plugin, along with the default cluster network
@@ -50,19 +48,18 @@ func NewCNI(cniBinDir string, cniConfigDir string) (*CniPlugin, error) {
 }
 
 func (cniParams *CNIParams) buildCNIRuntimeConf() *libcni.RuntimeConf {
-	logging.Verbosef("Pod name: %s; netns path %s; namespace: %s", cniParams.PodName, cniParams.NetnsPath, cniParams.Namespace)
+	logging.Verbosef("Pod name: %s; netns path %s; namespace: %s", cniParams.PodName, cniParams.CniCmdArgs.Netns, cniParams.Namespace)
 
 	return &libcni.RuntimeConf{
-		ContainerID: cniParams.SandboxID,
-		NetNS:       cniParams.NetnsPath,
-		IfName:      cniParams.IfaceName,
+		ContainerID: cniParams.CniCmdArgs.ContainerID,
+		NetNS:       cniParams.CniCmdArgs.Netns,
+		IfName:      cniParams.CniCmdArgs.IfName,
 		Args: [][2]string{
 			{"IgnoreUnknown", "1"},
 			{"K8S_POD_NAMESPACE", cniParams.Namespace},
 			{"K8S_POD_NAME", cniParams.PodName},
-			{"K8S_POD_INFRA_CONTAINER_ID", cniParams.SandboxID},
+			{"K8S_POD_INFRA_CONTAINER_ID", cniParams.CniCmdArgs.ContainerID},
 			{"K8S_POD_NETWORK", cniParams.NetworkName},
-			{"K8S_POD_IFMAC", cniParams.IfMAC},
 		},
 	}
 }

pkg/controller/pod.go

@@ -2,6 +2,9 @@ package controller
 
 import (
 	"fmt"
+	"github.com/containernetworking/cni/pkg/skel"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/k8sclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
 	"reflect"
 	"strings"
 	"time"
@@ -34,13 +37,15 @@ type PodNetworksController struct {
 	recorder         record.EventRecorder
 	containerRuntime containerruntimes.ContainerRuntime
 	cniPlugin        *cniclient.CniPlugin
+	confDir          string
 }
 
 // NewPodNetworksController returns new PodNetworksController instance
 func NewPodNetworksController(
 	k8sClientSet kubernetes.Interface,
 	podInformer cache.SharedIndexInformer,
-	cniPlugin *cniclient.CniPlugin) (*PodNetworksController, error) {
+	cniPlugin *cniclient.CniPlugin,
+	confDir string) (*PodNetworksController, error) {
 
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(logging.Verbosef)
@@ -60,6 +65,7 @@ func NewPodNetworksController(
 		recorder:         eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerName}),
 		containerRuntime: *containerRuntime,
 		cniPlugin:        cniPlugin,
+		confDir:          confDir,
 	}
 
 	podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -143,7 +149,7 @@ func (pnc *PodNetworksController) addNetworks(netsToAdd []types.NetworkSelection
 		if err != nil {
 			return logging.Errorf("failed to add network. error: %v", err)
 		}
-		logging.Verbosef("added network %s with iface name %s to pod %s; res: %+v", netToAdd.Name, cniParams.IfaceName, pod.GetName(), res)
+		logging.Verbosef("added network %s with iface name %s to pod %s; res: %+v", netToAdd.Name, cniParams.CniCmdArgs.IfName, pod.GetName(), res)
 	}
 	return nil
 }
@@ -155,10 +161,24 @@ func (pnc *PodNetworksController) removeNetworks(netsToRemove []types.NetworkSel
 			return logging.Errorf("failed to extract CNI params to remove existing interface from pod %s: %v", pod.GetName(), err)
 		}
 		logging.Verbosef("CNI params for pod %s: %+v", pod.GetName(), cniParams)
-		if err := pnc.cniPlugin.RemoveNetwork(*cniParams); err != nil {
+
+		k8sClient := &k8sclient.ClientInfo{
+			Client:           pnc.k8sClientSet,
+			NetClient:        nil,
+			EventBroadcaster: nil,
+			EventRecorder:    pnc.recorder,
+		}
+		delegateConf, _, err := k8sclient.GetKubernetesDelegate(k8sClient, &netToRemove, pnc.confDir, pod, nil)
+		if err != nil {
+			return fmt.Errorf("error retrieving the delegate info: %w", err)
+		}
+		k8sArgs, _ := k8sclient.GetK8sArgs(&cniParams.CniCmdArgs)
+		rtConf := &types.RuntimeConfig{}  // TODO: build the correct runtime config
+		multusNetconf := &types.NetConf{} // TODO: retrieve the multus config
+		if err := multus.DelPlugins(nil, pod, &cniParams.CniCmdArgs, k8sArgs, []*types.DelegateNetConf{delegateConf}, 1, rtConf, multusNetconf); err != nil {
 			return logging.Errorf("failed to remove network. error: %v", err)
 		}
-		logging.Verbosef("removed network %s from pod %s with interface name: %s", netToRemove.Name, pod.GetName(), cniParams.IfaceName)
+		logging.Verbosef("removed network %s from pod %s with interface name: %s", netToRemove.Name, pod.GetName(), cniParams.CniCmdArgs.IfName)
 	}
 	return nil
 }
@@ -227,14 +247,20 @@ func (pnc *PodNetworksController) getCNIParams(podObj *corev1.Pod, netSelectionE
 			// this *must* be defined.
 			interfaceName = "net1"
 		}
+
+		cmdArgs := skel.CmdArgs{
+			ContainerID: containerID,
+			Netns:       netns,
+			IfName:      interfaceName,
+			Args:        "",  // TODO: what to put here ?
+			Path:        "",  // TODO: what to put here ?
+			StdinData:   nil, // TODO: what to put here ?
+		}
 		return &cniclient.CNIParams{
 			Namespace:   namespace,
 			PodName:     podName,
-			SandboxID:   containerID,
-			NetnsPath:   netns,
 			NetworkName: netSelectionElement.Name,
-			IfMAC:       netSelectionElement.MacRequest,
-			IfaceName:   interfaceName,
+			CniCmdArgs:  cmdArgs,
 		}, nil
 	}
 	return nil, fmt.Errorf("failed to get pod %s container ID", podName)

pkg/k8sclient/k8sclient.go

@@ -130,42 +130,42 @@ func SetNetworkStatus(client *ClientInfo, k8sArgs *types.K8sArgs, netStatus []ne
 	return nil
 }
 
-func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElement, confdir string, pod *v1.Pod, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
+func GetKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElement, confdir string, pod *v1.Pod, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
 
-	logging.Debugf("getKubernetesDelegate: %v, %v, %s, %v, %v", client, net, confdir, pod, resourceMap)
+	logging.Debugf("GetKubernetesDelegate: %v, %v, %s, %v, %v", client, net, confdir, pod, resourceMap)
 	customResource, err := client.NetClient.NetworkAttachmentDefinitions(net.Namespace).Get(context.TODO(), net.Name, metav1.GetOptions{})
 	if err != nil {
 		errMsg := fmt.Sprintf("cannot find a network-attachment-definition (%s) in namespace (%s): %v", net.Name, net.Namespace, err)
 		if client != nil {
 			client.Eventf(pod, v1.EventTypeWarning, "NoNetworkFound", errMsg)
 		}
-		return nil, resourceMap, logging.Errorf("getKubernetesDelegate: " + errMsg)
+		return nil, resourceMap, logging.Errorf("GetKubernetesDelegate: " + errMsg)
 	}
 
 	// Get resourceName annotation from NetworkAttachmentDefinition
 	deviceID := ""
 	resourceName, ok := customResource.GetAnnotations()[resourceNameAnnot]
 	if ok && pod.Name != "" && pod.Namespace != "" {
 		// ResourceName annotation is found; try to get device info from resourceMap
-		logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName)
+		logging.Debugf("GetKubernetesDelegate: found resourceName annotation : %s", resourceName)
 
 		if resourceMap == nil {
 			ck, err := kubeletclient.GetResourceClient("")
 			if err != nil {
-				return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get a ResourceClient instance: %v", err)
+				return nil, resourceMap, logging.Errorf("GetKubernetesDelegate: failed to get a ResourceClient instance: %v", err)
 			}
 			resourceMap, err = ck.GetPodResourceMap(pod)
 			if err != nil {
-				return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from ResourceClient: %v", err)
+				return nil, resourceMap, logging.Errorf("GetKubernetesDelegate: failed to get resourceMap from ResourceClient: %v", err)
 			}
-			logging.Debugf("getKubernetesDelegate: resourceMap instance: %+v", resourceMap)
+			logging.Debugf("GetKubernetesDelegate: resourceMap instance: %+v", resourceMap)
 		}
 
 		entry, ok := resourceMap[resourceName]
 		if ok {
 			if idCount := len(entry.DeviceIDs); idCount > 0 && idCount > entry.Index {
 				deviceID = entry.DeviceIDs[entry.Index]
-				logging.Debugf("getKubernetesDelegate: podName: %s deviceID: %s", pod.Name, deviceID)
+				logging.Debugf("GetKubernetesDelegate: podName: %s deviceID: %s", pod.Name, deviceID)
 				entry.Index++ // increment Index for next delegate
 			}
 		}
@@ -366,7 +366,7 @@ func GetNetworkDelegates(k8sclient *ClientInfo, pod *v1.Pod, networks []*types.N
 			}
 		}
 
-		delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, conf.ConfDir, pod, resourceMap)
+		delegate, updatedResourceMap, err := GetKubernetesDelegate(k8sclient, net, conf.ConfDir, pod, resourceMap)
 		if err != nil {
 			return nil, logging.Errorf("GetNetworkDelegates: failed getting the delegate: %v", err)
 		}
@@ -393,7 +393,7 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 		Name:      netname,
 		Namespace: namespace,
 	}
-	delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap)
+	delegate, resourceMap, err := GetKubernetesDelegate(client, net, confdir, pod, resourceMap)
 	if err == nil {
 		return delegate, resourceMap, nil
 	}
@@ -497,7 +497,7 @@ func tryLoadK8sPodDefaultNetwork(kubeClient *ClientInfo, pod *v1.Pod, conf *type
 		return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: more than one default network is specified: %s", netAnnot)
 	}
 
-	delegate, _, err := getKubernetesDelegate(kubeClient, networks[0], conf.ConfDir, pod, nil)
+	delegate, _, err := GetKubernetesDelegate(kubeClient, networks[0], conf.ConfDir, pod, nil)
 	if err != nil {
 		return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: failed getting the delegate: %v", err)
 	}

pkg/k8sclient/k8sclient_test.go

@@ -157,7 +157,7 @@ var _ = Describe("k8sclient operations", func() {
 		netConf.ConfDir = tmpDir
 		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
 		Expect(len(delegates)).To(Equal(0))
-		Expect(err).To(MatchError("GetNetworkDelegates: failed getting the delegate: getKubernetesDelegate: cannot find a network-attachment-definition (net1) in namespace (test): network-attachment-definitions.k8s.cni.cncf.io \"net1\" not found"))
+		Expect(err).To(MatchError("GetNetworkDelegates: failed getting the delegate: GetKubernetesDelegate: cannot find a network-attachment-definition (net1) in namespace (test): network-attachment-definitions.k8s.cni.cncf.io \"net1\" not found"))
 	})
 
 	It("retrieves delegates from kubernetes using JSON format annotation", func() {
@@ -926,7 +926,7 @@ users:
 		})
 	})
 
-	Context("getKubernetesDelegate", func() {
+	Context("GetKubernetesDelegate", func() {
 		It("failed to get a ResourceClient instance", func() {
 			fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 			net1 := `{

pkg/multus/multus.go

@@ -458,13 +458,13 @@ func delegateDel(exec invoke.Exec, pod *v1.Pod, ifName string, delegateConf *typ
 	return err
 }
 
-// delPlugins deletes plugins in reverse order from lastdIdx
+// DelPlugins deletes plugins in reverse order from lastdIdx
 // Uses netRt as base RuntimeConf (coming from NetConf) but merges it
 // with each of the delegates' configuration
-func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *types.K8sArgs, delegates []*types.DelegateNetConf, lastIdx int, netRt *types.RuntimeConfig, multusNetconf *types.NetConf) error {
-	logging.Debugf("delPlugins: %v, %v, %v, %v, %v, %d, %v", exec, pod, args, k8sArgs, delegates, lastIdx, netRt)
+func DelPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *types.K8sArgs, delegates []*types.DelegateNetConf, lastIdx int, netRt *types.RuntimeConfig, multusNetconf *types.NetConf) error {
+	logging.Debugf("DelPlugins: %v, %v, %v, %v, %v, %d, %v", exec, pod, args, k8sArgs, delegates, lastIdx, netRt)
 	if os.Setenv("CNI_COMMAND", "DEL") != nil {
-		return logging.Errorf("delPlugins: error setting environment variable CNI_COMMAND to a value of DEL")
+		return logging.Errorf("DelPlugins: error setting environment variable CNI_COMMAND to a value of DEL")
 	}
 
 	var errorstrings []string
@@ -480,7 +480,7 @@ func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *type
 			// Even if the filename is set, file may not be present. Ignore error,
 			// but log and in the future may need to filter on specific errors.
 			if err != nil {
-				logging.Debugf("delPlugins: CleanDeviceInfoForCNI returned an error - err=%v", err)
+				logging.Debugf("DelPlugins: CleanDeviceInfoForCNI returned an error - err=%v", err)
 			}
 		}
 	}
@@ -645,7 +645,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 				netName = delegate.ConfList.Name
 			}
 			// Ignore errors; DEL must be idempotent anyway
-			_ = delPlugins(exec, nil, args, k8sArgs, n.Delegates, idx, n.RuntimeConfig, n)
+			_ = DelPlugins(exec, nil, args, k8sArgs, n.Delegates, idx, n.RuntimeConfig, n)
 			return nil, cmdPluginErr(k8sArgs, netName, "error adding container to network %q: %v", netName, err)
 		}
 
@@ -909,5 +909,5 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		}
 	}
 
-	return delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in)
+	return DelPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in)
 }