Support DeviceID on Auxiliary Bus

Device plugins may allocate network device on a bus different than PCI. sriov-network-device-plugin supports the allocation of network devices over Auxiliary bus[1][2][3]. extend host-device CNI to support such devices if provided through runtime config. - Check if device provided by DeviceID runtime config is present on either PCI bus or Auxiliary bus - extend getLink method to support getting netdev link obj from auxiliary bus - add unit-test to cover the new flow [1] https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin/tree/master?tab=readme-ov-file#auxiliary-network-devices-selectors [2] https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin/tree/master/docs/subfunctions [3] https://docs.kernel.org/networking/devlink/devlink-port.html Signed-off-by: adrianc <[email protected]>
containernetworking · Jan 24, 2024 · ecf00fc · ecf00fc
1 parent b6a0e0b
commit ecf00fc
Show file tree

Hide file tree

Showing 2 changed files with 124 additions and 31 deletions.
diff --git a/plugins/main/host-device/host-device.go b/plugins/main/host-device/host-device.go
@@ -37,7 +37,10 @@ import (
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
 
-var sysBusPCI = "/sys/bus/pci/devices"
+var (
+	sysBusPCI       = "/sys/bus/pci/devices"
+	sysBusAuxiliary = "/sys/bus/auxiliary/devices"
+)
 
 // Array of different linux drivers bound to network device needed for DPDK
 var userspaceDrivers = []string{"vfio-pci", "uio_pci_generic", "igb_uio"}
@@ -53,6 +56,9 @@ type NetConf struct {
 	RuntimeConfig struct {
 		DeviceID string `json:"deviceID,omitempty"`
 	} `json:"runtimeConfig,omitempty"`
+
+	// for internal use
+	AuxDevice string `json:"-"` // Auxiliary device name as appears on Auxiliary bus (/sys/bus/auxiliary)
 }
 
 func init() {
@@ -62,19 +68,44 @@ func init() {
 	runtime.LockOSThread()
 }
 
+// handleDeviceID updates netconf fields with DeviceID runtime config
+func handleDeviceID(netconf *NetConf) error {
+	deviceID := netconf.RuntimeConfig.DeviceID
+	if deviceID == "" {
+		return nil
+	}
+
+	// Check if deviceID is a PCI device
+	pciPath := filepath.Join(sysBusPCI, deviceID)
+	if _, err := os.Stat(pciPath); err == nil {
+		netconf.PCIAddr = deviceID
+		return nil
+	}
+
+	// Check if deviceID is an Auxiliary device
+	auxPath := filepath.Join(sysBusAuxiliary, deviceID)
+	if _, err := os.Stat(auxPath); err == nil {
+		netconf.PCIAddr = ""
+		netconf.AuxDevice = deviceID
+		return nil
+	}
+
+	return fmt.Errorf("runtime config DeviceID %s not found or unsupported", deviceID)
+}
+
 func loadConf(bytes []byte) (*NetConf, error) {
 	n := &NetConf{}
 	var err error
 	if err = json.Unmarshal(bytes, n); err != nil {
 		return nil, fmt.Errorf("failed to load netconf: %v", err)
 	}
 
-	if n.RuntimeConfig.DeviceID != "" {
-		// Override PCI device with the standardized DeviceID provided in Runtime Config.
-		n.PCIAddr = n.RuntimeConfig.DeviceID
+	// Override device with the standardized DeviceID if provided in Runtime Config.
+	if err := handleDeviceID(n); err != nil {
+		return nil, err
 	}
 
-	if n.Device == "" && n.HWAddr == "" && n.KernelPath == "" && n.PCIAddr == "" {
+	if n.Device == "" && n.HWAddr == "" && n.KernelPath == "" && n.PCIAddr == "" && n.AuxDevice == "" {
 		return nil, fmt.Errorf(`specify either "device", "hwaddr", "kernelpath" or "pciBusID"`)
 	}
 
@@ -102,7 +133,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 	result := &current.Result{}
 	var contDev netlink.Link
 	if !cfg.DPDKMode {
-		hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr)
+		hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr, cfg.AuxDevice)
 		if err != nil {
 			return fmt.Errorf("failed to find host device: %v", err)
 		}
@@ -314,11 +345,19 @@ func printLink(dev netlink.Link, cniVersion string, containerNs ns.NetNS) error
 	return types.PrintResult(&result, cniVersion)
 }
 
-func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error) {
-	links, err := netlink.LinkList()
+func linkFromPath(path string) (netlink.Link, error) {
+	entries, err := os.ReadDir(path)
 	if err != nil {
-		return nil, fmt.Errorf("failed to list node links: %v", err)
+		return nil, fmt.Errorf("failed to read directory %s: %q", path, err)
+	}
+	if len(entries) > 0 {
+		// grab the first net device
+		return netlink.LinkByName(entries[0].Name())
 	}
+	return nil, fmt.Errorf("failed to find network device in path %s", path)
+}
+
+func getLink(devname, hwaddr, kernelpath, pciaddr string, auxDev string) (netlink.Link, error) {
 	switch {
 
 	case len(devname) > 0:
@@ -329,6 +368,11 @@ func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error)
 			return nil, fmt.Errorf("failed to parse MAC address %q: %v", hwaddr, err)
 		}
 
+		links, err := netlink.LinkList()
+		if err != nil {
+			return nil, fmt.Errorf("failed to list node links: %v", err)
+		}
+
 		for _, link := range links {
 			if bytes.Equal(link.Attrs().HardwareAddr, hwAddr) {
 				return link, nil
@@ -339,20 +383,7 @@ func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error)
 			return nil, fmt.Errorf("kernel device path %q must be absolute and begin with /sys/devices/", kernelpath)
 		}
 		netDir := filepath.Join(kernelpath, "net")
-		entries, err := os.ReadDir(netDir)
-		if err != nil {
-			return nil, fmt.Errorf("failed to find network devices at %q", netDir)
-		}
-
-		// Grab the first device from eg /sys/devices/pci0000:00/0000:00:19.0/net
-		for _, entry := range entries {
-			// Make sure it's really an interface
-			for _, l := range links {
-				if entry.Name() == l.Attrs().Name {
-					return l, nil
-				}
-			}
-		}
+		return linkFromPath(netDir)
 	case len(pciaddr) > 0:
 		netDir := filepath.Join(sysBusPCI, pciaddr, "net")
 		if _, err := os.Lstat(netDir); err != nil {
@@ -363,14 +394,10 @@ func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error)
 			}
 			netDir = matches[0]
 		}
-		entries, err := os.ReadDir(netDir)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read net directory %s: %q", netDir, err)
-		}
-		if len(entries) > 0 {
-			return netlink.LinkByName(entries[0].Name())
-		}
-		return nil, fmt.Errorf("failed to find device name for pci address %s", pciaddr)
+		return linkFromPath(netDir)
+	case len(auxDev) > 0:
+		netDir := filepath.Join(sysBusAuxiliary, auxDev, "net")
+		return linkFromPath(netDir)
 	}
 
 	return nil, fmt.Errorf("failed to find physical interface")

diff --git a/plugins/main/host-device/host-device_test.go b/plugins/main/host-device/host-device_test.go
@@ -898,6 +898,71 @@ var _ = Describe("base functionality", func() {
 			})
 		})
 
+		It(fmt.Sprintf("Works with a valid %s config on auxiliary device", ver), func() {
+			var origLink netlink.Link
+			ifname := "eth0"
+
+			fs := &fakeFilesystem{
+				dirs: []string{
+					fmt.Sprintf("sys/bus/auxiliary/devices/mlx5_core.sf.4/net/%s", ifname),
+				},
+			}
+			defer fs.use()()
+
+			// prepare ifname in original namespace
+			_ = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				err := netlink.LinkAdd(&netlink.Dummy{
+					LinkAttrs: netlink.LinkAttrs{
+						Name: ifname,
+					},
+				})
+				Expect(err).NotTo(HaveOccurred())
+				origLink, err = netlink.LinkByName(ifname)
+				Expect(err).NotTo(HaveOccurred())
+				err = netlink.LinkSetUp(origLink)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+
+			// call CmdAdd
+			cniName := "net1"
+			conf := fmt.Sprintf(`{
+							"cniVersion": "%s",
+							"name": "cni-plugin-host-device-test",
+							"type": "host-device",
+							"runtimeConfig": {"deviceID": %q}
+						}`, ver, "mlx5_core.sf.4")
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				IfName:      cniName,
+				Netns:       targetNS.Path(),
+				StdinData:   []byte(conf),
+			}
+			var resI types.Result
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				var err error
+				resI, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) })
+				return err
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// check that the result was sane
+			t := newTesterByVersion(ver)
+			t.expectInterfaces(resI, cniName, origLink.Attrs().HardwareAddr.String(), targetNS.Path())
+
+			// call CmdDel
+			_ = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				err = testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+		})
+
 		It(fmt.Sprintf("Works with a valid %s config with IPAM", ver), func() {
 			var origLink netlink.Link
 
@@ -1181,6 +1246,7 @@ func (fs *fakeFilesystem) use() func() {
 	}
 
 	sysBusPCI = path.Join(fs.rootDir, "/sys/bus/pci/devices")
+	sysBusAuxiliary = path.Join(fs.rootDir, "/sys/bus/auxiliary/devices")
 
 	return func() {
 		// remove temporary fake fs