From e8488eef2341d5283e20f25d5680f03d96404df1 Mon Sep 17 00:00:00 2001 From: bohuini Date: Wed, 5 Jun 2024 13:12:22 -0700 Subject: [PATCH] Added NIC type and modified setRoutes function to support Windows --- cni/network/invoker_cns.go | 2 +- cns/NetworkContainerContract.go | 19 +- cns/middlewares/k8sSwiftV2.go | 208 ++++++------------ cns/middlewares/k8sSwiftV2_linux.go | 97 ++++++++ ...iftV2_test.go => k8sSwiftV2_test_linux.go} | 96 +++++++- cns/middlewares/k8sSwiftV2_windows.go | 12 + cns/middlewares/mock/mockClient.go | 87 +++++++- cns/middlewares/utils/utils.go | 19 +- crd/multitenancy/api/v1alpha1/utils.go | 28 +++ 9 files changed, 405 insertions(+), 163 deletions(-) create mode 100644 cns/middlewares/k8sSwiftV2_linux.go rename cns/middlewares/{k8sSwiftV2_test.go => k8sSwiftV2_test_linux.go} (73%) create mode 100644 cns/middlewares/k8sSwiftV2_windows.go create mode 100644 crd/multitenancy/api/v1alpha1/utils.go diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 54c5bca301..4f457aa93d 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -167,7 +167,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro key := invoker.getInterfaceInfoKey(info.nicType, info.macAddress) switch info.nicType { case cns.DelegatedVMNIC: - // only handling single v4 PodIPInfo for DelegatedVMNICs at the moment, will have to update once v6 gets added + // only handling single v4 PodIPInfo for Frontend NICs at the moment, will have to update once v6 gets added if !info.skipDefaultRoutes { numInterfacesWithDefaultRoutes++ } diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 11a4cbf25a..b49fbfe456 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -78,10 +78,18 @@ type NICType string // NIC Types const ( InfraNIC NICType = "InfraNIC" - // Delegated VM NICs are projected from VM to container network namespace - DelegatedVMNIC NICType = "DelegatedVMNIC" - // BackendNIC NICs are used for infiniband nics on a VM + // DelegatedVMNIC are projected from VM to container network namespace + DelegatedVMNIC NICType = "FrontendNIC" + // BackendNIC are used for infiniband NICs on a VM BackendNIC NICType = "BackendNIC" + // NodeNetworkInterfaceAccelnetFrontendNIC is a type of front-end nic that offers accelerated networking performance + NodeNetworkInterfaceAccelnetFrontendNIC NICType = "FrontendNIC_Accelnet" + + // TODO: These two const are currently unused due to version compatibility with DNC. DelegatedVMNIC and NodeNetworkInterfaceBackendNIC should be renamed to align with the naming convention with DNC + // NodeNetworkInterfaceFrontendNIC is the new name for DelegatedVMNIC + NodeNetworkInterfaceFrontendNIC NICType = "FrontendNIC" + // NodeNetworkInterfaceBackendNIC is the new name for BackendNIC + NodeNetworkInterfaceBackendNIC NICType = "BackendNIC" ) // ChannelMode :- CNS channel modes @@ -465,9 +473,8 @@ type PodIpInfo struct { PodIPConfig IPSubnet NetworkContainerPrimaryIPConfig IPConfiguration HostPrimaryIPInfo HostIPInfo - // NICType defines whether NIC is InfraNIC or DelegatedVMNIC or BackendNIC - NICType NICType - InterfaceName string + NICType NICType + InterfaceName string // MacAddress of interface MacAddress string // SkipDefaultRoutes is true if default routes should not be added on interface diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index a9721c3995..bf8919cf97 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -3,8 +3,6 @@ package middlewares import ( "context" "fmt" - "net/netip" - "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/logger" @@ -39,9 +37,9 @@ var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil) // IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request // and release IP configs handlers. -func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { +func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { - podInfo, respCode, message := m.validateIPConfigsRequest(ctx, &req) + podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req) if respCode != types.Success { return &cns.IPConfigsResponse{ @@ -49,7 +47,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa ReturnCode: respCode, Message: message, }, - }, errors.New("failed to validate ip configs request") + }, errors.New("failed to validate IP configs request") } ipConfigsResp, err := defaultHandler(ctx, req) // If the pod is not v2, return the response from the handler @@ -69,7 +67,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa if err != nil { return ipConfigsResp, err } - SWIFTv2PodIPInfo, err := m.getIPConfig(ctx, podInfo) + SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo) if err != nil { return &cns.IPConfigsResponse{ Response: cns.Response{ @@ -79,11 +77,11 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa PodIPInfo: []cns.PodIpInfo{}, }, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req) } - ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfo) + ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...) // Set routes for the pod for i := range ipConfigsResp.PodIPInfo { ipInfo := &ipConfigsResp.PodIPInfo[i] - err = m.setRoutes(ipInfo) + err = k.setRoutes(ipInfo) if err != nil { return &cns.IPConfigsResponse{ Response: cns.Response{ @@ -100,7 +98,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa // validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. // nolint -func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { +func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { @@ -110,7 +108,7 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod := v1.Pod{} - if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil { + if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil { errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) return nil, types.UnexpectedError, errBuf.Error() } @@ -121,11 +119,11 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() } // Check if the MTPNC CRD is ready. If one of the fields is empty, return error - if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + if !mtpnc.IsReady() { return nil, types.UnexpectedError, errMTPNCNotReady.Error() } } @@ -135,150 +133,78 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req } // getIPConfig returns the pod's SWIFT V2 IP configuration. -func (m *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { +func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { - return cns.PodIpInfo{}, errors.Wrapf(err, "failed to get pod's mtpnc from cache") + if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + return nil, errors.Wrapf(err, "failed to get pod's mtpnc from cache") } // Check if the MTPNC CRD is ready. If one of the fields is empty, return error - if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { - return cns.PodIpInfo{}, errMTPNCNotReady + if !mtpnc.IsReady() { + return nil, errMTPNCNotReady } logger.Printf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc) - // Parse MTPNC primaryIP to get the IP address and prefix length - p, err := netip.ParsePrefix(mtpnc.Status.PrimaryIP) - if err != nil { - return cns.PodIpInfo{}, errors.Wrapf(err, "failed to parse mtpnc primaryIP %s", mtpnc.Status.PrimaryIP) - } - // Get the IP address and prefix length - ip := p.Addr() - prefixSize := p.Bits() - if prefixSize != prefixLength { - return cns.PodIpInfo{}, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize) - } - podIPInfo := cns.PodIpInfo{ - PodIPConfig: cns.IPSubnet{ - IPAddress: ip.String(), - PrefixLength: uint8(prefixSize), - }, - MacAddress: mtpnc.Status.MacAddress, - NICType: cns.DelegatedVMNIC, - SkipDefaultRoutes: false, - // InterfaceName is empty for DelegatedVMNIC - } - - return podIPInfo, nil -} - -// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. -func (m *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { - logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType) - podIPInfo.Routes = []cns.Route{} - switch podIPInfo.NICType { - case cns.DelegatedVMNIC: - virtualGWRoute := cns.Route{ - IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength), - } - // default route via SWIFT v2 interface - route := cns.Route{ - IPAddress: "0.0.0.0/0", - GatewayIPAddress: virtualGW, - } - podIPInfo.Routes = []cns.Route{virtualGWRoute, route} - case cns.InfraNIC: - // Get and parse infraVNETCIDRs from env - infraVNETCIDRs, err := configuration.InfraVNETCIDRs() - if err != nil { - return errors.Wrapf(err, "failed to get infraVNETCIDRs from env") - } - infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse infraVNETCIDRs") - } - - // Get and parse podCIDRs from env - podCIDRs, err := configuration.PodCIDRs() - if err != nil { - return errors.Wrapf(err, "failed to get podCIDRs from env") - } - podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse podCIDRs") - } + var podIPInfos []cns.PodIpInfo - // Get and parse serviceCIDRs from env - serviceCIDRs, err := configuration.ServiceCIDRs() + if len(mtpnc.Status.InterfaceInfos) == 0 { + // Use fields from mtpnc.Status if InterfaceInfos is empty + ip, prefixSize, err := utils.ParseIPAndPrefix(mtpnc.Status.PrimaryIP) if err != nil { - return errors.Wrapf(err, "failed to get serviceCIDRs from env") - } - serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse serviceCIDRs") - } - // Check if the podIPInfo is IPv4 or IPv6 - ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) - if err != nil { - return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) - } - if ip.Is4() { - // routes for IPv4 podCIDR traffic - for _, podCIDRv4 := range podCIDRsV4 { - podCIDRv4Route := cns.Route{ - IPAddress: podCIDRv4, - GatewayIPAddress: overlayGatewayv4, - } - podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv4Route) - } - // route for IPv4 serviceCIDR traffic - for _, serviceCIDRv4 := range serviceCIDRsV4 { - serviceCIDRv4Route := cns.Route{ - IPAddress: serviceCIDRv4, - GatewayIPAddress: overlayGatewayv4, - } - podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) - } - // route for IPv4 infraVNETCIDR traffic - for _, infraVNETCIDRv4 := range infraVNETCIDRsv4 { - infraVNETCIDRv4Route := cns.Route{ - IPAddress: infraVNETCIDRv4, - GatewayIPAddress: overlayGatewayv4, - } - podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv4Route) + return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix") + } + if prefixSize != prefixLength { + return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize) + } + + podIPInfos = append(podIPInfos, cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: ip, + PrefixLength: uint8(prefixSize), + }, + MacAddress: mtpnc.Status.MacAddress, + NICType: cns.DelegatedVMNIC, + SkipDefaultRoutes: false, + // InterfaceName is empty for DelegatedVMNIC + }) + } else { + // Use InterfaceInfos if not empty + podIPInfos = make([]cns.PodIpInfo, len(mtpnc.Status.InterfaceInfos)) + for i, interfaceInfo := range mtpnc.Status.InterfaceInfos { + // Parse MTPNC primaryIP to get the IP address and prefix length + ip, prefixSize, err := utils.ParseIPAndPrefix(interfaceInfo.PrimaryIP) + if err != nil { + return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix") } - } else { - // routes for IPv6 podCIDR traffic - for _, podCIDRv6 := range podCIDRv6 { - podCIDRv6Route := cns.Route{ - IPAddress: podCIDRv6, - GatewayIPAddress: overlayGatewayV6, - } - podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv6Route) + if prefixSize != prefixLength { + return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize) } - // route for IPv6 serviceCIDR traffic - for _, serviceCIDRv6 := range serviceCIDRsV6 { - serviceCIDRv6Route := cns.Route{ - IPAddress: serviceCIDRv6, - GatewayIPAddress: overlayGatewayV6, - } - podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) + + var nicType cns.NICType + switch { + case interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC && !interfaceInfo.AccelnetEnabled: + nicType = cns.DelegatedVMNIC + case interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC && interfaceInfo.AccelnetEnabled: + nicType = cns.NodeNetworkInterfaceAccelnetFrontendNIC + case interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC: + nicType = cns.NodeNetworkInterfaceBackendNIC + default: + nicType = cns.DelegatedVMNIC } - // route for IPv6 infraVNETCIDR traffic - for _, infraVNETCIDRv6 := range infraVNETCIDRsv6 { - infraVNETCIDRv6Route := cns.Route{ - IPAddress: infraVNETCIDRv6, - GatewayIPAddress: overlayGatewayV6, - } - podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv6Route) + + podIPInfos[i] = cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: ip, + PrefixLength: uint8(prefixSize), + }, + MacAddress: interfaceInfo.MacAddress, + NICType: nicType, + SkipDefaultRoutes: false, } } - podIPInfo.SkipDefaultRoutes = true - case cns.BackendNIC: - default: - return errInvalidSWIFTv2NICType } - return nil + + return podIPInfos, nil } diff --git a/cns/middlewares/k8sSwiftV2_linux.go b/cns/middlewares/k8sSwiftV2_linux.go new file mode 100644 index 0000000000..c3b8f76c8c --- /dev/null +++ b/cns/middlewares/k8sSwiftV2_linux.go @@ -0,0 +1,97 @@ +package middlewares + +import ( + "fmt" + "net/netip" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares/utils" + "github.com/pkg/errors" +) + +// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. +func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { + logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType) + var routes []cns.Route + + switch podIPInfo.NICType { + case cns.DelegatedVMNIC: + virtualGWRoute := cns.Route{ + IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength), + } + // default route via SWIFT v2 interface + route := cns.Route{ + IPAddress: "0.0.0.0/0", + GatewayIPAddress: virtualGW, + } + routes = append(routes, virtualGWRoute, route) + + case cns.InfraNIC: + // Get and parse infraVNETCIDRs from env + infraVNETCIDRs, err := configuration.InfraVNETCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get infraVNETCIDRs from env") + } + infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse infraVNETCIDRs") + } + + // Get and parse podCIDRs from env + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get podCIDRs from env") + } + podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse podCIDRs") + } + + // Get and parse serviceCIDRs from env + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get serviceCIDRs from env") + } + serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse serviceCIDRs") + } + + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + + if ip.Is4() { + routes = append(routes, addRoutes(podCIDRsV4, overlayGatewayv4)...) + routes = append(routes, addRoutes(serviceCIDRsV4, overlayGatewayv4)...) + routes = append(routes, addRoutes(infraVNETCIDRsv4, overlayGatewayv4)...) + } else { + routes = append(routes, addRoutes(podCIDRv6, overlayGatewayV6)...) + routes = append(routes, addRoutes(serviceCIDRsV6, overlayGatewayV6)...) + routes = append(routes, addRoutes(infraVNETCIDRsv6, overlayGatewayV6)...) + } + podIPInfo.SkipDefaultRoutes = true + + case cns.BackendNIC, cns.NodeNetworkInterfaceBackendNIC, cns.NodeNetworkInterfaceAccelnetFrontendNIC: + // No-op NIC types. These NIC types are intentionally left as no-ops + default: + return errInvalidSWIFTv2NICType + } + + podIPInfo.Routes = routes + return nil +} + +func addRoutes(cidrs []string, gatewayIP string) []cns.Route { + routes := make([]cns.Route, len(cidrs)) + for i, cidr := range cidrs { + routes[i] = cns.Route{ + IPAddress: cidr, + GatewayIPAddress: gatewayIP, + } + } + return routes +} diff --git a/cns/middlewares/k8sSwiftV2_test.go b/cns/middlewares/k8sSwiftV2_test_linux.go similarity index 73% rename from cns/middlewares/k8sSwiftV2_test.go rename to cns/middlewares/k8sSwiftV2_test_linux.go index 856c4cd521..8a5fa2b0ca 100644 --- a/cns/middlewares/k8sSwiftV2_test.go +++ b/cns/middlewares/k8sSwiftV2_test_linux.go @@ -25,6 +25,15 @@ var ( testPod4GUID = "b21e1ee1-fb7e-4e6d-8c68-22ee5049944e" testPod4Info = cns.NewPodInfo("b21e1e-eth0", testPod4GUID, "testpod4", "testpod4namespace") + + testPod5GUID = "898fb8f1-f93e-4c96-9c31-6b89098949a3" + testPod5Info = cns.NewPodInfo("898fb8-eth0", testPod5GUID, "testpod5", "testpod5namespace") + + testPod6GUID = "898fb8f1-f93e-4c96-9c31-6b89098949a3" + testPod6Info = cns.NewPodInfo("898fb8-eth0", testPod6GUID, "testpod6", "testpod6namespace") + + testPod7GUID = "123e4567-e89b-12d3-a456-426614174000" + testPod7Info = cns.NewPodInfo("123e45-eth0", testPod7GUID, "testpod7", "testpod7namespace") ) func TestMain(m *testing.M) { @@ -179,10 +188,13 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} - ipInfo, err := middleware.getIPConfig(context.TODO(), testPod1Info) + ipInfos, err := middleware.getIPConfig(context.TODO(), testPod1Info) assert.Equal(t, err, nil) - assert.Equal(t, ipInfo.NICType, cns.DelegatedVMNIC) - assert.Equal(t, ipInfo.SkipDefaultRoutes, false) + // Ensure that the length of ipInfos matches the number of InterfaceInfos + // Adjust this according to the test setup + assert.Equal(t, len(ipInfos), 1) + assert.Equal(t, ipInfos[0].NICType, cns.DelegatedVMNIC) + assert.Equal(t, ipInfos[0].SkipDefaultRoutes, false) } func TestGetSWIFTv2IPConfigFailure(t *testing.T) { @@ -331,3 +343,81 @@ func TestSetRoutesFailure(t *testing.T) { } } } + +func TestAddRoutes(t *testing.T) { + cidrs := []string{"10.0.0.0/24", "20.0.0.0/24"} + gatewayIP := "192.168.1.1" + routes := addRoutes(cidrs, gatewayIP) + expectedRoutes := []cns.Route{ + { + IPAddress: "10.0.0.0/24", + GatewayIPAddress: gatewayIP, + }, + { + IPAddress: "20.0.0.0/24", + GatewayIPAddress: gatewayIP, + }, + } + assert.Equal(t, expectedRoutes, routes, "expected routes to match the expected routes") +} + +func TestNICTypeConfigSuccess(t *testing.T) { + middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} + + // Test Accelnet Frontend NIC type + ipInfos, err := middleware.getIPConfig(context.TODO(), testPod6Info) + assert.Equal(t, err, nil) + // Ensure that the length of ipInfos matches the number of InterfaceInfos + // Adjust this according to the test setup + assert.Equal(t, len(ipInfos), 1) + assert.Equal(t, ipInfos[0].NICType, cns.NodeNetworkInterfaceAccelnetFrontendNIC) + + // Test Backend NIC type + ipInfos2, err := middleware.getIPConfig(context.TODO(), testPod5Info) + assert.Equal(t, err, nil) + assert.Equal(t, len(ipInfos2), 1) + assert.Equal(t, ipInfos2[0].NICType, cns.BackendNIC) +} + +func TestGetSWIFTv2IPConfigMultiInterfaceFailure(t *testing.T) { + middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} + + // Pod's MTPNC doesn't exist in cache test + _, err := middleware.getIPConfig(context.TODO(), testPod3Info) + assert.ErrorContains(t, err, mock.ErrMTPNCNotFound.Error()) + + // Pod's MTPNC is not ready test + _, err = middleware.getIPConfig(context.TODO(), testPod4Info) + assert.Error(t, err, errMTPNCNotReady.Error()) +} + +func TestGetSWIFTv2IPConfigMultiInterfaceSuccess(t *testing.T) { + t.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") + t.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32") + t.Setenv(configuration.EnvInfraVNETCIDRs, "10.240.0.1/16,16A0:0020:AB00:0000::/32") + + middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} + + ipInfos, err := middleware.getIPConfig(context.TODO(), testPod7Info) + assert.Equal(t, err, nil) + // Ensure that the length of ipInfos matches the number of InterfaceInfos + // Adjust this according to the test setup in mock client + expectedInterfaceCount := 3 + assert.Equal(t, len(ipInfos), expectedInterfaceCount) + + for _, ipInfo := range ipInfos { + switch ipInfo.NICType { + case cns.DelegatedVMNIC: + assert.Equal(t, ipInfo.NICType, cns.DelegatedVMNIC) + case cns.NodeNetworkInterfaceAccelnetFrontendNIC: + assert.Equal(t, ipInfo.NICType, cns.NodeNetworkInterfaceAccelnetFrontendNIC) + case cns.NodeNetworkInterfaceBackendNIC: + assert.Equal(t, ipInfo.NICType, cns.NodeNetworkInterfaceBackendNIC) + case cns.InfraNIC: + assert.Equal(t, ipInfo.NICType, cns.InfraNIC) + default: + t.Errorf("unexpected NICType: %v", ipInfo.NICType) + } + assert.Equal(t, ipInfo.SkipDefaultRoutes, false) + } +} diff --git a/cns/middlewares/k8sSwiftV2_windows.go b/cns/middlewares/k8sSwiftV2_windows.go new file mode 100644 index 0000000000..361751584e --- /dev/null +++ b/cns/middlewares/k8sSwiftV2_windows.go @@ -0,0 +1,12 @@ +package middlewares + +import ( + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/logger" +) + +// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. This is a no-op as route setting is not applicable for Windows. +func (k *K8sSWIFTv2Middleware) setRoutes(_ *cns.PodIpInfo) error { + logger.Printf("[SWIFTv2Middleware] setRoutes is a no-op on Windows") + return nil +} diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index 2456def9fd..4c2b1adfa9 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -42,25 +42,87 @@ func NewClient() *Client { testPod4.Labels = make(map[string]string) testPod4.Labels[configuration.LabelPodSwiftV2] = podNetwork + testPod5 := v1.Pod{} + testPod5.Labels = make(map[string]string) + testPod5.Labels[configuration.LabelPodSwiftV2] = podNetwork + + testPod6 := v1.Pod{} + testPod6.Labels = make(map[string]string) + testPod6.Labels[configuration.LabelPodSwiftV2] = podNetwork + + testPod7 := v1.Pod{} + testPod7.Labels = make(map[string]string) + testPod7.Labels[configuration.LabelPodSwiftV2] = podNetwork + + testInterfaceInfos1 := v1alpha1.InterfaceInfo{ + NCID: "testncid", + PrimaryIP: "192.168.0.1/32", + MacAddress: "00:00:00:00:00:00", + GatewayIP: "10.0.0.1", + DeviceType: v1alpha1.DeviceTypeVnetNIC, + AccelnetEnabled: false, + } + testInterfaceInfos3 := v1alpha1.InterfaceInfo{ + NCID: "testncid", + PrimaryIP: "192.168.0.1/32", + MacAddress: "00:00:00:00:00:00", + GatewayIP: "10.0.0.1", + DeviceType: v1alpha1.DeviceTypeVnetNIC, + AccelnetEnabled: false, + } + testInterfaceInfos5 := v1alpha1.InterfaceInfo{ + NCID: "testncid", + PrimaryIP: "192.168.0.1/32", + MacAddress: "00:00:00:00:00:00", + GatewayIP: "10.0.0.1", + DeviceType: v1alpha1.DeviceTypeInfiniBandNIC, + AccelnetEnabled: true, + } + testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{ Status: v1alpha1.MultitenantPodNetworkConfigStatus{ - PrimaryIP: "192.168.0.1/32", - MacAddress: "00:00:00:00:00:00", - GatewayIP: "10.0.0.1", - NCID: "testncid", + InterfaceInfos: []v1alpha1.InterfaceInfo{testInterfaceInfos1}, }, } testMTPNC2 := v1alpha1.MultitenantPodNetworkConfig{} + testMTPNC3 := v1alpha1.MultitenantPodNetworkConfig{ + Status: v1alpha1.MultitenantPodNetworkConfigStatus{ + InterfaceInfos: []v1alpha1.InterfaceInfo{testInterfaceInfos3}, + }, + } + testMTPNC4 := v1alpha1.MultitenantPodNetworkConfig{} + testMTPNC5 := v1alpha1.MultitenantPodNetworkConfig{ + Status: v1alpha1.MultitenantPodNetworkConfigStatus{ + InterfaceInfos: []v1alpha1.InterfaceInfo{testInterfaceInfos5}, + }, + } + + testMTPNCMulti := v1alpha1.MultitenantPodNetworkConfig{ + Status: v1alpha1.MultitenantPodNetworkConfigStatus{ + InterfaceInfos: []v1alpha1.InterfaceInfo{testInterfaceInfos1, testInterfaceInfos3, testInterfaceInfos5}, + }, + } + return &Client{ - mtPodCache: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1, "testpod3namespace/testpod3": &testPod3, "testpod4namespace/testpod4": &testPod4}, + mtPodCache: map[string]*v1.Pod{ + "testpod1namespace/testpod1": &testPod1, + "testpod3namespace/testpod3": &testPod3, + "testpod4namespace/testpod4": &testPod4, + "testpod5namespace/testpod5": &testPod5, + "testpod6namespace/testpod6": &testPod6, + "testpod7namespace/testpod7": &testPod7, + }, mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{ "testpod1namespace/testpod1": &testMTPNC1, "testpod2namespace/testpod2": &testMTPNC2, "testpod4namespace/testpod4": &testMTPNC4, + "testpod5namespace/testpod5": &testMTPNC3, + "testpod6namespace/testpod6": &testMTPNC5, + "testpod7namespace/testpod7": &testMTPNCMulti, }, } } @@ -85,11 +147,18 @@ func (c *Client) Get(_ context.Context, key client.ObjectKey, obj client.Object, } func (c *Client) SetMTPNCReady() { + testInterfaceInfos1 := v1alpha1.InterfaceInfo{ + NCID: "testncid", + PrimaryIP: "192.168.0.1/32", + MacAddress: "00:00:00:00:00:00", + GatewayIP: "10.0.0.1", + DeviceType: v1alpha1.DeviceTypeVnetNIC, + AccelnetEnabled: false, + } + testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} - testMTPNC1.Status.PrimaryIP = "192.168.0.1/32" - testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" - testMTPNC1.Status.GatewayIP = "10.0.0.1" - testMTPNC1.Status.NCID = "testncid" + testMTPNC1.Status.InterfaceInfos = []v1alpha1.InterfaceInfo{testInterfaceInfos1} + c.mtpncCache["testpod1namespace/testpod1"] = &testMTPNC1 } diff --git a/cns/middlewares/utils/utils.go b/cns/middlewares/utils/utils.go index 2eca8551ea..687e96a86a 100644 --- a/cns/middlewares/utils/utils.go +++ b/cns/middlewares/utils/utils.go @@ -1,19 +1,20 @@ package utils import ( - "fmt" "net/netip" "strings" + + "github.com/pkg/errors" ) -// ParseCIDRs parses the comma separated list of CIDRs and returns the IPv4 and IPv6 CIDRs. +// ParseCIDRs parses the comma-separated list of CIDRs and returns the IPv4 and IPv6 CIDRs. func ParseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { v4IPs = []string{} v6IPs = []string{} for _, cidr := range strings.Split(cidrs, ",") { p, err := netip.ParsePrefix(cidr) if err != nil { - return nil, nil, fmt.Errorf("failed to parse CIDR %s : %w", cidr, err) + return nil, nil, errors.Wrapf(err, "failed to parse CIDR %s", cidr) } ip := p.Addr() if ip.Is4() { @@ -24,3 +25,15 @@ func ParseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { } return v4IPs, v6IPs, nil } + +// ParseIPAndPrefix parses the primaryIP and returns the IP address and prefix length. +func ParseIPAndPrefix(primaryIP string) (ip string, prefixSize int, err error) { + p, err := netip.ParsePrefix(primaryIP) + if err != nil { + err = errors.Wrapf(err, "failed to parse IP %s", primaryIP) + return "", 0, err + } + ip = p.Addr().String() + prefixSize = p.Bits() + return ip, prefixSize, nil +} diff --git a/crd/multitenancy/api/v1alpha1/utils.go b/crd/multitenancy/api/v1alpha1/utils.go new file mode 100644 index 0000000000..28e3a51163 --- /dev/null +++ b/crd/multitenancy/api/v1alpha1/utils.go @@ -0,0 +1,28 @@ +package v1alpha1 + +// IsReady checks if all the required fields in the MTPNC status are populated +func (m *MultitenantPodNetworkConfig) IsReady() bool { + // Check if InterfaceInfos slice is not empty + if len(m.Status.InterfaceInfos) == 0 { + // Check if the higher fields in the status are populated + if m.Status.PrimaryIP == "" || + m.Status.MacAddress == "" || + m.Status.NCID == "" || + m.Status.GatewayIP == "" { + return false + } + } else { + // Check if each InterfaceInfo has all required fields populated + for _, interfaceInfo := range m.Status.InterfaceInfos { + if interfaceInfo.NCID == "" || + interfaceInfo.PrimaryIP == "" || + interfaceInfo.MacAddress == "" || + interfaceInfo.GatewayIP == "" || + interfaceInfo.DeviceType == "" { + return false + } + } + } + + return true +}