From 3b32b9c1b8fd7fae3f14d287b50347524f90e455 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Mon, 18 Mar 2024 13:08:59 -0700 Subject: [PATCH 001/102] ci: changes up to endpointInternal --- cni/network/invoker.go | 1 + cni/network/network.go | 32 ++++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 9d0ed8444e..8a27114c8b 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -30,6 +30,7 @@ type IPAMAddResult struct { // Splitting defaultInterfaceInfo from secondaryInterfacesInfo so we don't need to loop for default CNI result every time defaultInterfaceInfo network.InterfaceInfo secondaryInterfacesInfo []network.InterfaceInfo + interfaceInfo []network.InterfaceInfo // ncResponse is used for Swift 1.0 multitenancy ncResponse *cns.GetNetworkContainerResponse hostSubnetPrefix net.IPNet diff --git a/cni/network/network.go b/cni/network/network.go index 2381b8ffc4..33a579bc77 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -457,7 +457,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if len(ipamAddResults) > 1 && !plugin.isDualNicFeatureSupported(args.Netns) { errMsg := fmt.Sprintf("received multiple NC results %+v from CNS while dualnic feature is not supported", ipamAddResults) logger.Error("received multiple NC results from CNS while dualnic feature is not supported", - zap.Any("results", ipamAddResult)) + zap.Any("results", ipamAddResult)) // <- should this be Result(S)? return plugin.Errorf(errMsg) } } else { @@ -471,6 +471,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { for i := 0; i < len(ipamAddResults); i++ { var networkID string ipamAddResult = ipamAddResults[i] + defaultIndex := findDefaultInterface(ipamAddResult) options := make(map[string]any) networkID, err = plugin.getNetworkName(args.Netns, &ipamAddResult, nwCfg) @@ -500,7 +501,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } if resultSecondAdd != nil { - ipamAddResult.defaultInterfaceInfo = convertCniResultToInterfaceInfo(resultSecondAdd) + ipamAddResult.interfaceInfo[defaultIndex] = convertCniResultToInterfaceInfo(resultSecondAdd) return nil } } @@ -518,18 +519,20 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} if !nwCfg.MultiTenancy { + // what does this .Add() do? ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig) if err != nil { return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } - sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.defaultInterfaceInfo, ipamAddResult.secondaryInterfacesInfo)) + // This proably needs to be changed as we return all interfaces... + sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[defaultIndex], ipamAddResult.interfaceInfo)) } defer func() { //nolint:gocritic if err != nil { // for multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips if !(nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS) { - plugin.cleanupAllocationOnError(ipamAddResult.defaultInterfaceInfo.IPConfigs, nwCfg, args, options) + plugin.cleanupAllocationOnError(ipamAddResult.interfaceInfo[defaultIndex].IPConfigs, nwCfg, args, options) } } }() @@ -540,7 +543,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { logger.Info("Creating network", zap.String("networkID", networkID)) sendEvent(plugin, fmt.Sprintf("[cni-net] Creating network %v.", networkID)) // opts map needs to get passed in here - if nwInfo, err = plugin.createNetworkInternal(networkID, policies, ipamAddConfig, ipamAddResult); err != nil { + if nwInfo, err = plugin.createNetworkInternal(networkID, policies, ipamAddConfig, ipamAddResult, defaultIndex); err != nil { logger.Error("Create network failed", zap.Error(err)) return err } @@ -603,6 +606,7 @@ func (plugin *NetPlugin) createNetworkInternal( policies []policy.Policy, ipamAddConfig IPAMAddConfig, ipamAddResult IPAMAddResult, + defaultIndex int, ) (network.NetworkInfo, error) { nwInfo := network.NetworkInfo{} ipamAddResult.hostSubnetPrefix.IP = ipamAddResult.hostSubnetPrefix.IP.Mask(ipamAddResult.hostSubnetPrefix.Mask) @@ -649,7 +653,7 @@ func (plugin *NetPlugin) createNetworkInternal( IsIPv6Enabled: ipamAddResult.ipv6Enabled, } - if err = addSubnetToNetworkInfo(ipamAddResult, &nwInfo); err != nil { + if err = addSubnetToNetworkInfo(ipamAddResult, &nwInfo, defaultIndex); err != nil { logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) return nwInfo, err @@ -665,8 +669,8 @@ func (plugin *NetPlugin) createNetworkInternal( } // construct network info with ipv4/ipv6 subnets -func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.NetworkInfo) error { - for _, ipConfig := range ipamAddResult.defaultInterfaceInfo.IPConfigs { +func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.NetworkInfo, defaultIndex int) error { + for _, ipConfig := range ipamAddResult.interfaceInfo[defaultIndex].IPConfigs { ip, podSubnetPrefix, err := net.ParseCIDR(ipConfig.Address.String()) if err != nil { return fmt.Errorf("Failed to ParseCIDR for pod subnet prefix: %w", err) @@ -1404,3 +1408,15 @@ func convertCniResultToInterfaceInfo(result *cniTypesCurr.Result) network.Interf return interfaceInfo } + +func findDefaultInterface(ipamAddResult IPAMAddResult) int { + // defaultIf := network.InterfaceInfo{} + for i, ifInfo := range ipamAddResult.interfaceInfo { + if ifInfo.NICType == cns.InfraNIC { + return i + } + } + + // error or nil here + return 0 +} From a7647d7e335c472943ceab14f03c9225d4196ec1 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Mon, 18 Mar 2024 13:47:54 -0700 Subject: [PATCH 002/102] ci: remove defaultInterface from invoker --- cni/network/invoker_cns.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index f3e75690f7..74b991807c 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -384,15 +384,15 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } } + result := network.InterfaceInfo{} if ip := net.ParseIP(info.podIPAddress); ip != nil { - defaultInterfaceInfo := &addResult.defaultInterfaceInfo defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix if ip.To4() == nil { defaultRouteDstPrefix = network.Ipv6DefaultRouteDstPrefix addResult.ipv6Enabled = true } - defaultInterfaceInfo.IPConfigs = append(defaultInterfaceInfo.IPConfigs, + result.IPConfigs = append(result.IPConfigs, &network.IPConfig{ Address: net.IPNet{ IP: ip, @@ -407,15 +407,15 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } if len(routes) > 0 { - defaultInterfaceInfo.Routes = append(defaultInterfaceInfo.Routes, routes...) + result.Routes = append(result.Routes, routes...) } else { // add default routes if none are provided - defaultInterfaceInfo.Routes = append(defaultInterfaceInfo.Routes, network.RouteInfo{ + result.Routes = append(result.Routes, network.RouteInfo{ Dst: defaultRouteDstPrefix, Gw: ncgw, }) } - addResult.defaultInterfaceInfo.SkipDefaultRoutes = info.skipDefaultRoutes + result.SkipDefaultRoutes = info.skipDefaultRoutes } // get the name of the primary IP address @@ -425,7 +425,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } addResult.hostSubnetPrefix = *hostIPNet - addResult.defaultInterfaceInfo.NICType = cns.InfraNIC + result.NICType = cns.InfraNIC // set subnet prefix for host vm // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode @@ -436,6 +436,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } } + addResult.interfaceInfo = append(addResult.interfaceInfo, result) return nil } @@ -470,7 +471,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p SkipDefaultRoutes: info.skipDefaultRoutes, } - addResult.secondaryInterfacesInfo = append(addResult.secondaryInterfacesInfo, result) + addResult.interfaceInfo = append(addResult.interfaceInfo, result) return nil } From ee9300bb9bdafb1d65bbc40a191abff62a318ff3 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Mon, 18 Mar 2024 14:02:33 -0700 Subject: [PATCH 003/102] ci: change up to CreateEndpoint --- cni/network/network.go | 47 +++++++++++++++++++++++++----------------- network/manager.go | 8 +++---- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 33a579bc77..9e65a274f2 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -572,7 +572,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } var epInfo network.EndpointInfo - epInfo, err = plugin.createEndpointInternal(&createEndpointInternalOpt) + epInfo, err = plugin.createEndpointInternal(&createEndpointInternalOpt, defaultIndex) if err != nil { logger.Error("Endpoint creation failed", zap.Error(err)) return err @@ -707,10 +707,10 @@ type createEndpointInternalOpt struct { natInfo []policy.NATInfo } -func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) (network.EndpointInfo, error) { +func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, defaultIndex int) (network.EndpointInfo, error) { epInfo := network.EndpointInfo{} - defaultInterfaceInfo := opt.ipamAddResult.defaultInterfaceInfo + defaultInterfaceInfo := opt.ipamAddResult.interfaceInfo[defaultIndex] // can also call function, but this will be faster epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultInterfaceInfo.DNS, opt.k8sNamespace) if err != nil { err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) @@ -758,7 +758,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) ServiceCidrs: opt.nwCfg.ServiceCidrs, NATInfo: opt.natInfo, NICType: cns.InfraNIC, - SkipDefaultRoutes: opt.ipamAddResult.defaultInterfaceInfo.SkipDefaultRoutes, + SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[defaultIndex].SkipDefaultRoutes, Routes: defaultInterfaceInfo.Routes, } @@ -797,22 +797,31 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) epInfos := []*network.EndpointInfo{&epInfo} // get secondary interface info - for _, secondaryCniResult := range opt.ipamAddResult.secondaryInterfacesInfo { - var addresses []net.IPNet - for _, ipconfig := range secondaryCniResult.IPConfigs { - addresses = append(addresses, ipconfig.Address) - } + for _, secondaryCniResult := range opt.ipamAddResult.interfaceInfo { + switch secondaryCniResult.NICType { + case cns.DelegatedVMNIC: + // secondary + var addresses []net.IPNet + for _, ipconfig := range secondaryCniResult.IPConfigs { + addresses = append(addresses, ipconfig.Address) + } - epInfos = append(epInfos, - &network.EndpointInfo{ - ContainerID: epInfo.ContainerID, - NetNsPath: epInfo.NetNsPath, - IPAddresses: addresses, - Routes: secondaryCniResult.Routes, - MacAddress: secondaryCniResult.MacAddress, - NICType: secondaryCniResult.NICType, - SkipDefaultRoutes: secondaryCniResult.SkipDefaultRoutes, - }) + epInfos = append(epInfos, + &network.EndpointInfo{ + ContainerID: epInfo.ContainerID, + NetNsPath: epInfo.NetNsPath, + IPAddresses: addresses, + Routes: secondaryCniResult.Routes, + MacAddress: secondaryCniResult.MacAddress, + NICType: secondaryCniResult.NICType, + SkipDefaultRoutes: secondaryCniResult.SkipDefaultRoutes, + }) + case cns.BackendNIC: + // todo + default: + // InfraNic + continue + } } // Create the endpoint. diff --git a/network/manager.go b/network/manager.go index d0b6c90713..0f283e89b5 100644 --- a/network/manager.go +++ b/network/manager.go @@ -372,7 +372,7 @@ func (nm *networkManager) GetNetworkInfo(networkId string) (NetworkInfo, error) } // CreateEndpoint creates a new container endpoint. -func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo []*EndpointInfo) error { +func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo []*EndpointInfo, defaultIndex int) error { nm.Lock() defer nm.Unlock() @@ -382,10 +382,10 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } if nw.VlanId != 0 { - // the first entry in epInfo is InfraNIC type - if epInfo[0].Data[VlanIDKey] == nil { + // leverage passing default index, if failed, call function to find default + if epInfo[defaultIndex].Data[VlanIDKey] == nil { logger.Info("overriding endpoint vlanid with network vlanid") - epInfo[0].Data[VlanIDKey] = nw.VlanId + epInfo[defaultIndex].Data[VlanIDKey] = nw.VlanId } } From c3ba7cf6bcf8babc1ddd7785461cdc81d98b3120 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Mon, 18 Mar 2024 17:52:58 -0700 Subject: [PATCH 004/102] ci: changes up to CreateEndpoint() --- network/endpoint.go | 6 ++++-- network/endpoint_linux.go | 3 ++- network/endpoint_windows.go | 7 ++++--- network/manager.go | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index 19202c8be7..2f0d51afca 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -143,19 +143,21 @@ func (nw *network) newEndpoint( nsc NamespaceClientInterface, iptc ipTablesClient, epInfo []*EndpointInfo, + defaultIndex int, ) (*endpoint, error) { var ep *endpoint var err error defer func() { if err != nil { - logger.Error("Failed to create endpoint with err", zap.String("id", epInfo[0].Id), zap.Error(err)) + // this log is hardcoded for default + logger.Error("Failed to create endpoint with err", zap.String("id", epInfo[defaultIndex].Id), zap.Error(err)) } }() // Call the platform implementation. // Pass nil for epClient and will be initialized in newendpointImpl - ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, nsc, iptc, epInfo) + ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, nsc, iptc, epInfo, defaultIndex) if err != nil { return nil, err } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 179d28cc8c..fcdb5379d8 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -58,6 +58,7 @@ func (nw *network) newEndpointImpl( nsc NamespaceClientInterface, iptc ipTablesClient, epInfo []*EndpointInfo, + _ int, ) (*endpoint, error) { var ( err error @@ -65,7 +66,7 @@ func (nw *network) newEndpointImpl( contIfName string localIP string vlanid = 0 - defaultEpInfo = epInfo[0] + defaultEpInfo = epInfo[0] // this needs to be changed, out of scope... containerIf *net.Interface ) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index c0ddf486bf..8aee0f867a 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -74,17 +74,18 @@ func (nw *network) newEndpointImpl( _ NamespaceClientInterface, _ ipTablesClient, epInfo []*EndpointInfo, + defaultIndex int, ) (*endpoint, error) { // there is only 1 epInfo for windows, multiple interfaces will be added in the future - if useHnsV2, err := UseHnsV2(epInfo[0].NetNsPath); useHnsV2 { + if useHnsV2, err := UseHnsV2(epInfo[defaultIndex].NetNsPath); useHnsV2 { if err != nil { return nil, err } - return nw.newEndpointImplHnsV2(cli, epInfo[0]) + return nw.newEndpointImplHnsV2(cli, epInfo[defaultIndex]) } - return nw.newEndpointImplHnsV1(epInfo[0], plc) + return nw.newEndpointImplHnsV1(epInfo[defaultIndex], plc) } // newEndpointImplHnsV1 creates a new endpoint in the network using HnsV1 diff --git a/network/manager.go b/network/manager.go index 0f283e89b5..dd6a3c7998 100644 --- a/network/manager.go +++ b/network/manager.go @@ -389,7 +389,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } } - ep, err := nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, nm.iptablesClient, epInfo) + ep, err := nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, nm.iptablesClient, epInfo, defaultIndex) if err != nil { return err } From a97d7983d23b1d825658ff706bbe908f7346df12 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 19 Mar 2024 12:01:09 -0700 Subject: [PATCH 005/102] ci: invoker cns and UT fixes --- cni/network/invoker_cns.go | 26 +++++++++++++++--------- cni/network/invoker_cns_test.go | 36 +++++++++++++++++++++++++-------- cni/network/invoker_mock.go | 7 ++++--- cni/network/network.go | 4 ++-- cni/network/network_test.go | 6 +++--- network/manager.go | 2 +- network/manager_mock.go | 4 ++-- 7 files changed, 56 insertions(+), 29 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 74b991807c..e10e8d1433 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -143,7 +143,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro addResult := IPAMAddResult{} numInterfacesWithDefaultRoutes := 0 - + defaultIndex := -1 for i := 0; i < len(response.PodIPInfo); i++ { info := IPResultInfo{ podIPAddress: response.PodIPInfo[i].PodIPConfig.IPAddress, @@ -175,9 +175,16 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro return IPAMAddResult{}, err } default: + // Assign default index + if defaultIndex == -1 { + // add a default Interface + // would put within configureDefaultAddResult() but we have a dependency on numInterfacesWithDefaultRoutes + addResult.interfaceInfo = append(addResult.interfaceInfo, network.InterfaceInfo{NICType: cns.InfraNIC}) + defaultIndex = len(addResult.interfaceInfo) - 1 + } // only count dualstack interface once - if addResult.defaultInterfaceInfo.IPConfigs == nil { - addResult.defaultInterfaceInfo.IPConfigs = make([]*network.IPConfig, 0) + if addResult.interfaceInfo[defaultIndex].IPConfigs == nil { + addResult.interfaceInfo[defaultIndex].IPConfigs = make([]*network.IPConfig, 0) if !info.skipDefaultRoutes { numInterfacesWithDefaultRoutes++ } @@ -384,7 +391,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } } - result := network.InterfaceInfo{} + defaultIndex := findDefaultInterface(*addResult) if ip := net.ParseIP(info.podIPAddress); ip != nil { defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix if ip.To4() == nil { @@ -392,7 +399,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add addResult.ipv6Enabled = true } - result.IPConfigs = append(result.IPConfigs, + addResult.interfaceInfo[defaultIndex].IPConfigs = append(addResult.interfaceInfo[defaultIndex].IPConfigs, &network.IPConfig{ Address: net.IPNet{ IP: ip, @@ -407,15 +414,15 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } if len(routes) > 0 { - result.Routes = append(result.Routes, routes...) + addResult.interfaceInfo[defaultIndex].Routes = append(addResult.interfaceInfo[defaultIndex].Routes, routes...) } else { // add default routes if none are provided - result.Routes = append(result.Routes, network.RouteInfo{ + addResult.interfaceInfo[defaultIndex].Routes = append(addResult.interfaceInfo[defaultIndex].Routes, network.RouteInfo{ Dst: defaultRouteDstPrefix, Gw: ncgw, }) } - result.SkipDefaultRoutes = info.skipDefaultRoutes + addResult.interfaceInfo[defaultIndex].SkipDefaultRoutes = info.skipDefaultRoutes } // get the name of the primary IP address @@ -425,7 +432,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } addResult.hostSubnetPrefix = *hostIPNet - result.NICType = cns.InfraNIC + addResult.interfaceInfo[defaultIndex].NICType = cns.InfraNIC // This can be removed // set subnet prefix for host vm // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode @@ -436,7 +443,6 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } } - addResult.interfaceInfo = append(addResult.interfaceInfo, result) return nil } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index b81456d2aa..1e6da50994 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -485,10 +485,20 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ipamAddResult.secondaryInterfacesInfo) - require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo, "incorrect default response") - if len(tt.wantSecondaryInterfacesInfo.IPConfigs) > 0 { - require.EqualValues(tt.wantSecondaryInterfacesInfo, ipamAddResult.secondaryInterfacesInfo[0], "incorrect multitenant response") + for _, ifInfo := range ipamAddResult.interfaceInfo { + switch ifInfo.NICType { + case cns.DelegatedVMNIC: + // Secondary + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ifInfo) + if len(tt.wantSecondaryInterfacesInfo.IPConfigs) > 0 { + require.EqualValues(tt.wantSecondaryInterfacesInfo, ifInfo, "incorrect multitenant response") + } + case cns.BackendNIC: + // todo + default: + // Default | InfraNIC + require.Equalf(tt.wantDefaultResult, ifInfo, "incorrect default response") + } } }) } @@ -713,10 +723,20 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo) - require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo, "incorrect default response") - if len(tt.wantMultitenantResult.IPConfigs) > 0 { - require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo[0], "incorrect multitenant response") + for _, ifInfo := range ipamAddResult.interfaceInfo { + switch ifInfo.NICType { + case cns.DelegatedVMNIC: + // Secondary + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ifInfo) + if len(tt.wantMultitenantResult.IPConfigs) > 0 { + require.Equalf(tt.wantMultitenantResult, ifInfo, "incorrect multitenant response") + } + case cns.BackendNIC: + // todo + default: + // Default | InfraNIC + require.Equalf(tt.wantDefaultResult, ifInfo, "incorrect default response") + } } }) } diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 5bc4ad5d01..b1c23b9264 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -60,7 +60,7 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv4Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetBits, ipv4Bits)} gwIP := net.ParseIP("10.240.0.1") - ipamAddResult.defaultInterfaceInfo = network.InterfaceInfo{ + result := network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ {Address: ipnet, Gateway: gwIP}, }, @@ -80,9 +80,10 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv6Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetv6Bits, ipv6Bits)} gwIP := net.ParseIP("fc00::1") - ipamAddResult.defaultInterfaceInfo.IPConfigs = append(ipamAddResult.defaultInterfaceInfo.IPConfigs, &network.IPConfig{Address: ipnet, Gateway: gwIP}) + result.IPConfigs = append(ipamAddResult.defaultInterfaceInfo.IPConfigs, &network.IPConfig{Address: ipnet, Gateway: gwIP}) invoker.ipMap[ipnet.String()] = true } + ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, result) if invoker.delegatedVMNIC { if invoker.delegatedVMNICFail { @@ -91,7 +92,7 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ipStr := "20.20.20.20/32" _, ipnet, _ := net.ParseCIDR(ipStr) - ipamAddResult.secondaryInterfacesInfo = append(ipamAddResult.secondaryInterfacesInfo, network.InterfaceInfo{ + ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ {Address: *ipnet}, }, diff --git a/cni/network/network.go b/cni/network/network.go index 9e65a274f2..b09c47a30b 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -827,7 +827,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, // Create the endpoint. logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) - err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, epInfos) + err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, epInfos, defaultIndex) if err != nil { err = plugin.Errorf("Failed to create endpoint: %v", err) } @@ -1427,5 +1427,5 @@ func findDefaultInterface(ipamAddResult IPAMAddResult) int { } // error or nil here - return 0 + return -1 } diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 0df2010a8b..a8f87e36be 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -1042,13 +1042,13 @@ func TestGetAllEndpointState(t *testing.T) { ep2 := getTestEndpoint("podname2", "podnamespace2", "10.0.0.2/24", "podinterfaceid2", "testcontainerid2") ep3 := getTestEndpoint("podname3", "podnamespace3", "10.240.1.242/16", "podinterfaceid3", "testcontainerid3") - err := plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep1}) + err := plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep1}, 0) // giving zero to test UT, probably need to change require.NoError(t, err) - err = plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep2}) + err = plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep2}, 0) require.NoError(t, err) - err = plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep3}) + err = plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep3}, 0) require.NoError(t, err) state, err := plugin.GetAllEndpointState(networkid) diff --git a/network/manager.go b/network/manager.go index dd6a3c7998..adf98c4ed0 100644 --- a/network/manager.go +++ b/network/manager.go @@ -98,7 +98,7 @@ type NetworkManager interface { FindNetworkIDFromNetNs(netNs string) (string, error) GetNumEndpointsByContainerID(containerID string) int - CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo) error + CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo, defaultIndex int) error DeleteEndpoint(networkID string, endpointID string, epInfo *EndpointInfo) error GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) diff --git a/network/manager_mock.go b/network/manager_mock.go index 316af8d5d3..d490799122 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -53,13 +53,13 @@ func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (NetworkInfo, err } // CreateEndpoint mock -func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfos []*EndpointInfo) error { +func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfos []*EndpointInfo, _ int) error { for _, epInfo := range epInfos { if err := nm.TestEndpointClient.AddEndpoints(epInfo); err != nil { return err } } - + // Look into changing this... nm.TestEndpointInfoMap[epInfos[0].Id] = epInfos[0] return nil } From 30f646ba965163b7892d656ff63b3ea20de3952e Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 19 Mar 2024 12:44:51 -0700 Subject: [PATCH 006/102] ci: add fixes to UT(s), capture non populated defaultInterface failures --- cni/network/invoker_cns_test.go | 15 +++++++++++++-- cni/network/network.go | 9 ++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 1e6da50994..49cf714464 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -767,7 +767,7 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { fields fields args args want network.InterfaceInfo - want1 network.InterfaceInfo + want1 network.InterfaceInfo // We dont use this anywhere..? Dead Code potentially, ask jaeryn wantErr bool }{ { @@ -853,7 +853,18 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { t.Fatalf("expected an error %+v but none received", err) } require.NoError(err) - require.Equalf(tt.want, ipamAddResult.defaultInterfaceInfo, "incorrect ipv4 response") + + for _, ifInfo := range ipamAddResult.interfaceInfo { + switch ifInfo.NICType { + case cns.DelegatedVMNIC: + // Secondary + case cns.BackendNIC: + // todo + default: + // Default | InfraNIC + require.Equalf(tt.want, ifInfo, "incorrect ipv4 response") + } + } }) } } diff --git a/cni/network/network.go b/cni/network/network.go index b09c47a30b..3c35da2704 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -362,7 +362,11 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { telemetry.SendCNIMetric(&cniMetric, plugin.tb) // Add Interfaces to result. - defaultCniResult := convertInterfaceInfoToCniResult(ipamAddResult.defaultInterfaceInfo, args.IfName) + if len(ipamAddResult.interfaceInfo) < 1 { + ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{NICType: cns.InfraNIC}) + } + defaultIndex := findDefaultInterface(ipamAddResult) + defaultCniResult := convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[defaultIndex], args.IfName) addSnatInterface(nwCfg, defaultCniResult) @@ -524,6 +528,9 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } + if defaultIndex == -1 { + defaultIndex = findDefaultInterface(ipamAddResult) + } // This proably needs to be changed as we return all interfaces... sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[defaultIndex], ipamAddResult.interfaceInfo)) } From 2048eb7b39af520b2372459638bd613a0bcdd246 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 19 Mar 2024 14:00:28 -0700 Subject: [PATCH 007/102] ci: multitenancy changes --- cni/network/multitenancy.go | 9 +++++---- cni/network/multitenancy_mock.go | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 33d161b78b..dcb0f0ae10 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -216,14 +216,15 @@ func (m *Multitenancy) GetAllNetworkContainers( } ipamResults := make([]IPAMAddResult, len(ncResponses)) - + // Can use hard coded 0 as ipamResults is empty and we are creating the first interface for i := 0; i < len(ncResponses); i++ { ipamResults[i].ncResponse = &ncResponses[i] ipamResults[i].hostSubnetPrefix = hostSubnetPrefixes[i] ipconfig, routes := convertToIPConfigAndRouteInfo(ipamResults[i].ncResponse) - ipamResults[i].defaultInterfaceInfo.IPConfigs = []*network.IPConfig{ipconfig} - ipamResults[i].defaultInterfaceInfo.Routes = routes - ipamResults[i].defaultInterfaceInfo.NICType = cns.InfraNIC + ipamResults[i].interfaceInfo = append(ipamResults[i].interfaceInfo, network.InterfaceInfo{}) + ipamResults[i].interfaceInfo[0].IPConfigs = []*network.IPConfig{ipconfig} + ipamResults[i].interfaceInfo[0].Routes = routes + ipamResults[i].interfaceInfo[0].NICType = cns.InfraNIC } return ipamResults, err diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index 9cd6b0a5ed..bf4497eb75 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -155,13 +155,15 @@ func (m *MockMultitenancy) GetAllNetworkContainers( cnsResponses = append(cnsResponses, *cnsResponseOne) ipamResults := make([]IPAMAddResult, len(cnsResponses)) + // Can use hard coded 0 as ipamResults is empty and we are creating the first interface for i := 0; i < len(cnsResponses); i++ { ipamResults[i].ncResponse = &cnsResponses[i] ipamResults[i].hostSubnetPrefix = ipNets[i] ipconfig, routes := convertToIPConfigAndRouteInfo(ipamResults[i].ncResponse) - ipamResults[i].defaultInterfaceInfo.IPConfigs = []*network.IPConfig{ipconfig} - ipamResults[i].defaultInterfaceInfo.Routes = routes - ipamResults[i].defaultInterfaceInfo.NICType = cns.InfraNIC + ipamResults[i].interfaceInfo = append(ipamResults[i].interfaceInfo, network.InterfaceInfo{}) + ipamResults[i].interfaceInfo[0].IPConfigs = []*network.IPConfig{ipconfig} + ipamResults[i].interfaceInfo[0].Routes = routes + ipamResults[i].interfaceInfo[0].NICType = cns.InfraNIC } return ipamResults, nil From ed5d5f9285a9cd3e493a71b7b70540ee3cd04f38 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 19 Mar 2024 15:28:05 -0700 Subject: [PATCH 008/102] ci: invoker azure changes & remove all defaultInterfaceInfo possible --- cni/network/invoker_azure.go | 9 +++++---- cni/network/invoker_azure_test.go | 14 ++++++++++++-- cni/network/invoker_mock.go | 8 ++++---- cni/network/network.go | 20 ++++++++++++-------- cni/network/network_windows.go | 6 +++--- cni/network/network_windows_test.go | 10 +++++----- 6 files changed, 41 insertions(+), 26 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index d453c3de18..07af6cd5c9 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -75,8 +75,10 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er defer func() { if err != nil { - if len(addResult.defaultInterfaceInfo.IPConfigs) > 0 { - if er := invoker.Delete(&addResult.defaultInterfaceInfo.IPConfigs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { + // defaultIndex := findDefaultInterface(addResult) + // Can hard code 0 here as it everything is new. Afriad of creating leaks if I complicate the logic + if len(addResult.interfaceInfo[0].IPConfigs) > 0 { + if er := invoker.Delete(&addResult.interfaceInfo[0].IPConfigs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { err = invoker.plugin.Errorf("Failed to clean up IP's during Delete with error %v, after Add failed with error %w", er, err) } } else { @@ -116,8 +118,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er routes[i] = network.RouteInfo{Dst: route.Dst, Gw: route.GW} } - addResult.defaultInterfaceInfo = network.InterfaceInfo{IPConfigs: ipconfigs, Routes: routes, DNS: network.DNSInfo{Suffix: result.DNS.Domain, Servers: result.DNS.Nameservers}, NICType: cns.InfraNIC} - + addResult.interfaceInfo = append(addResult.interfaceInfo, network.InterfaceInfo{IPConfigs: ipconfigs, Routes: routes, DNS: network.DNSInfo{Suffix: result.DNS.Domain, Servers: result.DNS.Nameservers}, NICType: cns.InfraNIC}) return addResult, err } diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 2fe6bcb5e6..ebd752338d 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -238,8 +238,18 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { require.Nil(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ipamAddResult.defaultInterfaceInfo.IPConfigs) - require.Exactly(tt.want, ipamAddResult.defaultInterfaceInfo.IPConfigs) + for _, ifInfo := range ipamAddResult.interfaceInfo { + switch string(ifInfo.NICType) { + case "DelegatedVMNIC": + // Secondary + case "BackendNIC": + // todo + default: + // Default | InfraNIC + fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ifInfo.IPConfigs) + require.Exactly(tt.want, ifInfo.IPConfigs) + } + } }) } } diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index b1c23b9264..6178aef63b 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -60,17 +60,18 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv4Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetBits, ipv4Bits)} gwIP := net.ParseIP("10.240.0.1") - result := network.InterfaceInfo{ + ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ {Address: ipnet, Gateway: gwIP}, }, NICType: cns.InfraNIC, - } + }) invoker.ipMap[ipnet.String()] = true if invoker.v6Fail { return ipamAddResult, errV6 } + defaultIndex := findDefaultInterface(ipamAddResult) if invoker.isIPv6 { ipv6Str := "fc00::2" if _, ok := invoker.ipMap["fc00::2/128"]; ok { @@ -80,10 +81,9 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv6Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetv6Bits, ipv6Bits)} gwIP := net.ParseIP("fc00::1") - result.IPConfigs = append(ipamAddResult.defaultInterfaceInfo.IPConfigs, &network.IPConfig{Address: ipnet, Gateway: gwIP}) + ipamAddResult.interfaceInfo[defaultIndex].IPConfigs = append(ipamAddResult.interfaceInfo[defaultIndex].IPConfigs, &network.IPConfig{Address: ipnet, Gateway: gwIP}) invoker.ipMap[ipnet.String()] = true } - ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, result) if invoker.delegatedVMNIC { if invoker.delegatedVMNICFail { diff --git a/cni/network/network.go b/cni/network/network.go index 3c35da2704..e6a12a4c7e 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -417,6 +417,10 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { res, err = plugin.nnsClient.AddContainerNetworking(context.Background(), k8sPodName, args.Netns) if err == nil { + // Need to figure out what the NICtype is. Currently there is defer() code that has a -1 index error as part of logic if an infraNic does not exist. + // ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{ + // IPConfigs: convertNnsToIPConfigs(res, args.IfName, k8sPodName, "AddContainerNetworking"), + // }) ipamAddResult.defaultInterfaceInfo.IPConfigs = convertNnsToIPConfigs(res, args.IfName, k8sPodName, "AddContainerNetworking") } @@ -586,7 +590,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", - ipamAddResult.defaultInterfaceInfo.IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + ipamAddResult.interfaceInfo[defaultIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) } return nil @@ -633,7 +637,7 @@ func (plugin *NetPlugin) createNetworkInternal( return nwInfo, err } - nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.defaultInterfaceInfo.DNS) + nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.interfaceInfo[defaultIndex].DNS) if err != nil { err = plugin.Errorf("Failed to getDNSSettings: %v", err) return nwInfo, err @@ -717,8 +721,8 @@ type createEndpointInternalOpt struct { func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, defaultIndex int) (network.EndpointInfo, error) { epInfo := network.EndpointInfo{} - defaultInterfaceInfo := opt.ipamAddResult.interfaceInfo[defaultIndex] // can also call function, but this will be faster - epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultInterfaceInfo.DNS, opt.k8sNamespace) + ifInfo := opt.ipamAddResult.interfaceInfo[defaultIndex] // can also call function, but this will be faster + epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, ifInfo.DNS, opt.k8sNamespace) if err != nil { err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) return epInfo, err @@ -726,7 +730,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, policyArgs := PolicyArgs{ nwInfo: opt.nwInfo, nwCfg: opt.nwCfg, - ipconfigs: defaultInterfaceInfo.IPConfigs, + ipconfigs: ifInfo.IPConfigs, } endpointPolicies, err := getEndpointPolicies(policyArgs) if err != nil { @@ -766,7 +770,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, NATInfo: opt.natInfo, NICType: cns.InfraNIC, SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[defaultIndex].SkipDefaultRoutes, - Routes: defaultInterfaceInfo.Routes, + Routes: ifInfo.Routes, } epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) @@ -777,7 +781,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, epInfo.Policies = append(epInfo.Policies, epPolicies...) // Populate addresses. - for _, ipconfig := range defaultInterfaceInfo.IPConfigs { + for _, ipconfig := range ifInfo.IPConfigs { epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) } @@ -790,7 +794,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, } if opt.nwCfg.MultiTenancy { - plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, &defaultInterfaceInfo) + plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, &ifInfo) } setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index ba0fdfaf4a..39b496ba3a 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -157,8 +157,10 @@ func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResu // First try to build the network name from the cnsResponse if present // This will happen during ADD call if ipamAddResult != nil && ipamAddResult.ncResponse != nil { + // find defaultInterface within AddResult + defaultIndex := findDefaultInterface(ipamAddResult) // networkName will look like ~ azure-vlan1-172-28-1-0_24 - ipAddrNet := ipamAddResult.defaultInterfaceInfo.IPConfigs[0].Address + ipAddrNet := ipamAddResult.interfaceInfo[defaultIndex].IPConfigs[0].Address prefix, err := netip.ParsePrefix(ipAddrNet.String()) if err != nil { logger.Error("Error parsing network CIDR", @@ -284,7 +286,6 @@ func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig, isIPv6Enabled bool) ([] Protocol: protocol, Flags: flag, }) - if err != nil { return nil, errors.Wrap(err, "failed to marshal HNS portMappingPolicySetting") } @@ -293,7 +294,6 @@ func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig, isIPv6Enabled bool) ([] Type: hnsv2.PortMapping, Settings: rawPolicy, }) - if err != nil { return nil, errors.Wrap(err, "failed to marshal HNS endpointPolicy") } diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index eaa61f4264..94aa3094e8 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -366,7 +366,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - defaultInterfaceInfo: network.InterfaceInfo{ + interfaceInfo: []network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ { Address: net.IPNet{ @@ -401,7 +401,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - defaultInterfaceInfo: network.InterfaceInfo{ + interfaceInfo: []network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ { Address: net.IPNet{ @@ -436,7 +436,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - defaultInterfaceInfo: network.InterfaceInfo{ + interfaceInfo: []network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ { Address: net.IPNet{ @@ -471,7 +471,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - defaultInterfaceInfo: network.InterfaceInfo{ + interfaceInfo: []network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ { Address: net.IPNet{ @@ -502,7 +502,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, ipamAddResult: &IPAMAddResult{ ncResponse: &cns.GetNetworkContainerResponse{}, - defaultInterfaceInfo: network.InterfaceInfo{ + interfaceInfo: []network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ { Address: net.IPNet{ From e6763141f4f0e3d95266f8dd25e99b433b6449b9 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 19 Mar 2024 16:55:42 -0700 Subject: [PATCH 009/102] ci add NICType to baremetal flow --- cni/network/network.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index e6a12a4c7e..f93c8b1491 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -417,11 +417,10 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { res, err = plugin.nnsClient.AddContainerNetworking(context.Background(), k8sPodName, args.Netns) if err == nil { - // Need to figure out what the NICtype is. Currently there is defer() code that has a -1 index error as part of logic if an infraNic does not exist. - // ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{ - // IPConfigs: convertNnsToIPConfigs(res, args.IfName, k8sPodName, "AddContainerNetworking"), - // }) - ipamAddResult.defaultInterfaceInfo.IPConfigs = convertNnsToIPConfigs(res, args.IfName, k8sPodName, "AddContainerNetworking") + ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{ + IPConfigs: convertNnsToIPConfigs(res, args.IfName, k8sPodName, "AddContainerNetworking"), + NICType: cns.InfraNIC, + }) } return err From ef86f032849d4f9aead5d4b3665f4c371897a76d Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 19 Mar 2024 16:58:45 -0700 Subject: [PATCH 010/102] chore: address comments --- cni/network/invoker.go | 4 +- cni/network/invoker_azure.go | 12 +++- cni/network/invoker_azure_test.go | 15 ++--- cni/network/invoker_cns.go | 27 +++++---- cni/network/invoker_cns_test.go | 30 +++++----- cni/network/invoker_mock.go | 10 +++- cni/network/network.go | 91 ++++++++++++++++------------- cni/network/network_windows.go | 9 ++- cni/network/network_windows_test.go | 60 +++++++++++-------- cnm/network/network.go | 3 +- network/endpoint.go | 7 +-- network/endpoint_linux.go | 4 +- network/endpoint_test.go | 18 +++--- network/endpoint_windows.go | 8 +-- network/manager.go | 11 ++-- network/manager_mock.go | 2 +- 16 files changed, 176 insertions(+), 135 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 8a27114c8b..8a47ebc02f 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -28,9 +28,7 @@ type IPAMAddConfig struct { type IPAMAddResult struct { // Splitting defaultInterfaceInfo from secondaryInterfacesInfo so we don't need to loop for default CNI result every time - defaultInterfaceInfo network.InterfaceInfo - secondaryInterfacesInfo []network.InterfaceInfo - interfaceInfo []network.InterfaceInfo + interfaceInfo []network.InterfaceInfo // ncResponse is used for Swift 1.0 multitenancy ncResponse *cns.GetNetworkContainerResponse hostSubnetPrefix net.IPNet diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index 07af6cd5c9..c61d3ce1b7 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -75,8 +75,6 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er defer func() { if err != nil { - // defaultIndex := findDefaultInterface(addResult) - // Can hard code 0 here as it everything is new. Afriad of creating leaks if I complicate the logic if len(addResult.interfaceInfo[0].IPConfigs) > 0 { if er := invoker.Delete(&addResult.interfaceInfo[0].IPConfigs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { err = invoker.plugin.Errorf("Failed to clean up IP's during Delete with error %v, after Add failed with error %w", er, err) @@ -118,7 +116,15 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er routes[i] = network.RouteInfo{Dst: route.Dst, Gw: route.GW} } - addResult.interfaceInfo = append(addResult.interfaceInfo, network.InterfaceInfo{IPConfigs: ipconfigs, Routes: routes, DNS: network.DNSInfo{Suffix: result.DNS.Domain, Servers: result.DNS.Nameservers}, NICType: cns.InfraNIC}) + addResult.interfaceInfo = append(addResult.interfaceInfo, network.InterfaceInfo{ + IPConfigs: ipconfigs, + Routes: routes, + DNS: network.DNSInfo{ + Suffix: result.DNS.Domain, + Servers: result.DNS.Nameservers, + }, + NICType: cns.InfraNIC, + }) return addResult, err } diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index ebd752338d..1ca9fd43ac 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -8,6 +8,7 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cni/log" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/ipam" "github.com/Azure/azure-container-networking/network" cniSkel "github.com/containernetworking/cni/pkg/skel" @@ -239,15 +240,15 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { } for _, ifInfo := range ipamAddResult.interfaceInfo { - switch string(ifInfo.NICType) { - case "DelegatedVMNIC": - // Secondary - case "BackendNIC": - // todo - default: - // Default | InfraNIC + switch ifInfo.NICType { + case cns.DelegatedVMNIC, cns.BackendNIC: + fmt.Print(errInvalidNIC) + require.FailNow("No coverage for this NICType") + case cns.InfraNIC: fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ifInfo.IPConfigs) require.Exactly(tt.want, ifInfo.IPConfigs) + default: + require.FailNow("Unsupported NICType") } } }) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index e10e8d1433..46b86a803d 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -143,7 +143,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro addResult := IPAMAddResult{} numInterfacesWithDefaultRoutes := 0 - defaultIndex := -1 + ifIndex := -1 for i := 0; i < len(response.PodIPInfo); i++ { info := IPResultInfo{ podIPAddress: response.PodIPInfo[i].PodIPConfig.IPAddress, @@ -176,15 +176,15 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } default: // Assign default index - if defaultIndex == -1 { + if ifIndex == -1 { // add a default Interface // would put within configureDefaultAddResult() but we have a dependency on numInterfacesWithDefaultRoutes addResult.interfaceInfo = append(addResult.interfaceInfo, network.InterfaceInfo{NICType: cns.InfraNIC}) - defaultIndex = len(addResult.interfaceInfo) - 1 + ifIndex = len(addResult.interfaceInfo) - 1 } // only count dualstack interface once - if addResult.interfaceInfo[defaultIndex].IPConfigs == nil { - addResult.interfaceInfo[defaultIndex].IPConfigs = make([]*network.IPConfig, 0) + if addResult.interfaceInfo[ifIndex].IPConfigs == nil { + addResult.interfaceInfo[ifIndex].IPConfigs = make([]*network.IPConfig, 0) if !info.skipDefaultRoutes { numInterfacesWithDefaultRoutes++ } @@ -391,7 +391,12 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } } - defaultIndex := findDefaultInterface(*addResult) + ifIndex, err := findDefaultInterface(*addResult) + if err != nil { + logger.Error("Error finding InfraNIC interface", + zap.Error(err)) + return errors.Wrap(err, "error finding InfraNIC interface") + } if ip := net.ParseIP(info.podIPAddress); ip != nil { defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix if ip.To4() == nil { @@ -399,7 +404,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add addResult.ipv6Enabled = true } - addResult.interfaceInfo[defaultIndex].IPConfigs = append(addResult.interfaceInfo[defaultIndex].IPConfigs, + addResult.interfaceInfo[ifIndex].IPConfigs = append(addResult.interfaceInfo[ifIndex].IPConfigs, &network.IPConfig{ Address: net.IPNet{ IP: ip, @@ -414,15 +419,15 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } if len(routes) > 0 { - addResult.interfaceInfo[defaultIndex].Routes = append(addResult.interfaceInfo[defaultIndex].Routes, routes...) + addResult.interfaceInfo[ifIndex].Routes = append(addResult.interfaceInfo[ifIndex].Routes, routes...) } else { // add default routes if none are provided - addResult.interfaceInfo[defaultIndex].Routes = append(addResult.interfaceInfo[defaultIndex].Routes, network.RouteInfo{ + addResult.interfaceInfo[ifIndex].Routes = append(addResult.interfaceInfo[ifIndex].Routes, network.RouteInfo{ Dst: defaultRouteDstPrefix, Gw: ncgw, }) } - addResult.interfaceInfo[defaultIndex].SkipDefaultRoutes = info.skipDefaultRoutes + addResult.interfaceInfo[ifIndex].SkipDefaultRoutes = info.skipDefaultRoutes } // get the name of the primary IP address @@ -432,7 +437,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } addResult.hostSubnetPrefix = *hostIPNet - addResult.interfaceInfo[defaultIndex].NICType = cns.InfraNIC // This can be removed + addResult.interfaceInfo[ifIndex].NICType = cns.InfraNIC // This can be removed // set subnet prefix for host vm // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 49cf714464..c6cc1d767c 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -488,16 +488,17 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { for _, ifInfo := range ipamAddResult.interfaceInfo { switch ifInfo.NICType { case cns.DelegatedVMNIC: - // Secondary fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ifInfo) if len(tt.wantSecondaryInterfacesInfo.IPConfigs) > 0 { require.EqualValues(tt.wantSecondaryInterfacesInfo, ifInfo, "incorrect multitenant response") } case cns.BackendNIC: - // todo - default: - // Default | InfraNIC + fmt.Print(errInvalidNIC) + require.FailNow("No coverage for this NICType") + case cns.InfraNIC: require.Equalf(tt.wantDefaultResult, ifInfo, "incorrect default response") + default: + require.FailNow("Unsupported NICType") } } }) @@ -726,16 +727,17 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { for _, ifInfo := range ipamAddResult.interfaceInfo { switch ifInfo.NICType { case cns.DelegatedVMNIC: - // Secondary fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ifInfo) if len(tt.wantMultitenantResult.IPConfigs) > 0 { require.Equalf(tt.wantMultitenantResult, ifInfo, "incorrect multitenant response") } case cns.BackendNIC: - // todo - default: - // Default | InfraNIC + fmt.Print(errInvalidNIC) + require.FailNow("No coverage for this NICType") + case cns.InfraNIC: require.Equalf(tt.wantDefaultResult, ifInfo, "incorrect default response") + default: + require.FailNow("Unsupported NICType") } } }) @@ -856,13 +858,13 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { for _, ifInfo := range ipamAddResult.interfaceInfo { switch ifInfo.NICType { - case cns.DelegatedVMNIC: - // Secondary - case cns.BackendNIC: - // todo - default: - // Default | InfraNIC + case cns.DelegatedVMNIC, cns.BackendNIC: + fmt.Print(errInvalidNIC) + require.FailNow("No coverage for this NICType") + case cns.InfraNIC: require.Equalf(tt.want, ifInfo, "incorrect ipv4 response") + default: + require.FailNow("Unsupported NICType") } } }) diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 6178aef63b..4946ddd39d 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -21,6 +21,8 @@ var ( errV4 = errors.New("v4 fail") errV6 = errors.New("v6 Fail") errDelegatedVMNIC = errors.New("delegatedVMNIC fail") + errNoInfraNIC = errors.New("no InfraNIC fail") + errInvalidNIC = errors.New("no test case for this NIC") errDeleteIpam = errors.New("delete fail") ) @@ -71,7 +73,11 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes return ipamAddResult, errV6 } - defaultIndex := findDefaultInterface(ipamAddResult) + ifIndex, err := findDefaultInterface(ipamAddResult) + if err != nil { + return IPAMAddResult{}, errNoInfraNIC + } + if invoker.isIPv6 { ipv6Str := "fc00::2" if _, ok := invoker.ipMap["fc00::2/128"]; ok { @@ -81,7 +87,7 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv6Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetv6Bits, ipv6Bits)} gwIP := net.ParseIP("fc00::1") - ipamAddResult.interfaceInfo[defaultIndex].IPConfigs = append(ipamAddResult.interfaceInfo[defaultIndex].IPConfigs, &network.IPConfig{Address: ipnet, Gateway: gwIP}) + ipamAddResult.interfaceInfo[ifIndex].IPConfigs = append(ipamAddResult.interfaceInfo[ifIndex].IPConfigs, &network.IPConfig{Address: ipnet, Gateway: gwIP}) invoker.ipMap[ipnet.String()] = true } diff --git a/cni/network/network.go b/cni/network/network.go index f93c8b1491..7ac66caab6 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -362,11 +362,12 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { telemetry.SendCNIMetric(&cniMetric, plugin.tb) // Add Interfaces to result. - if len(ipamAddResult.interfaceInfo) < 1 { - ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{NICType: cns.InfraNIC}) + ifIndex, _ := findDefaultInterface(ipamAddResult) + if ifIndex < 0 { + ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{}) + ifIndex = len(ipamAddResult.interfaceInfo) - 1 } - defaultIndex := findDefaultInterface(ipamAddResult) - defaultCniResult := convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[defaultIndex], args.IfName) + defaultCniResult := convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[ifIndex], args.IfName) addSnatInterface(nwCfg, defaultCniResult) @@ -464,7 +465,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if len(ipamAddResults) > 1 && !plugin.isDualNicFeatureSupported(args.Netns) { errMsg := fmt.Sprintf("received multiple NC results %+v from CNS while dualnic feature is not supported", ipamAddResults) logger.Error("received multiple NC results from CNS while dualnic feature is not supported", - zap.Any("results", ipamAddResult)) // <- should this be Result(S)? + zap.Any("results", ipamAddResult)) return plugin.Errorf(errMsg) } } else { @@ -478,7 +479,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { for i := 0; i < len(ipamAddResults); i++ { var networkID string ipamAddResult = ipamAddResults[i] - defaultIndex := findDefaultInterface(ipamAddResult) + ifIndex, _ := findDefaultInterface(ipamAddResult) options := make(map[string]any) networkID, err = plugin.getNetworkName(args.Netns, &ipamAddResult, nwCfg) @@ -508,7 +509,12 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } if resultSecondAdd != nil { - ipamAddResult.interfaceInfo[defaultIndex] = convertCniResultToInterfaceInfo(resultSecondAdd) + ifIndex, err = findDefaultInterface(ipamAddResult) + if err != nil { + ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{}) + ifIndex = len(ipamAddResult.interfaceInfo) - 1 + } + ipamAddResult.interfaceInfo[ifIndex] = convertCniResultToInterfaceInfo(resultSecondAdd) return nil } } @@ -526,23 +532,25 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} if !nwCfg.MultiTenancy { - // what does this .Add() do? ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig) if err != nil { return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } - if defaultIndex == -1 { - defaultIndex = findDefaultInterface(ipamAddResult) + + ifIndex, err = findDefaultInterface(ipamAddResult) + if err != nil { + logger.Error("Error finding InfraNIC interface", + zap.Error(err)) + return errors.Wrap(err, "error finding InfraNIC interface") } - // This proably needs to be changed as we return all interfaces... - sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[defaultIndex], ipamAddResult.interfaceInfo)) + sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) } defer func() { //nolint:gocritic if err != nil { // for multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips if !(nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS) { - plugin.cleanupAllocationOnError(ipamAddResult.interfaceInfo[defaultIndex].IPConfigs, nwCfg, args, options) + plugin.cleanupAllocationOnError(ipamAddResult.interfaceInfo[ifIndex].IPConfigs, nwCfg, args, options) } } }() @@ -553,7 +561,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { logger.Info("Creating network", zap.String("networkID", networkID)) sendEvent(plugin, fmt.Sprintf("[cni-net] Creating network %v.", networkID)) // opts map needs to get passed in here - if nwInfo, err = plugin.createNetworkInternal(networkID, policies, ipamAddConfig, ipamAddResult, defaultIndex); err != nil { + if nwInfo, err = plugin.createNetworkInternal(networkID, policies, ipamAddConfig, ipamAddResult, ifIndex); err != nil { logger.Error("Create network failed", zap.Error(err)) return err } @@ -582,14 +590,14 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } var epInfo network.EndpointInfo - epInfo, err = plugin.createEndpointInternal(&createEndpointInternalOpt, defaultIndex) + epInfo, err = plugin.createEndpointInternal(&createEndpointInternalOpt, ifIndex) if err != nil { logger.Error("Endpoint creation failed", zap.Error(err)) return err } sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", - ipamAddResult.interfaceInfo[defaultIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) } return nil @@ -616,7 +624,7 @@ func (plugin *NetPlugin) createNetworkInternal( policies []policy.Policy, ipamAddConfig IPAMAddConfig, ipamAddResult IPAMAddResult, - defaultIndex int, + ifIndex int, ) (network.NetworkInfo, error) { nwInfo := network.NetworkInfo{} ipamAddResult.hostSubnetPrefix.IP = ipamAddResult.hostSubnetPrefix.IP.Mask(ipamAddResult.hostSubnetPrefix.Mask) @@ -636,7 +644,7 @@ func (plugin *NetPlugin) createNetworkInternal( return nwInfo, err } - nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.interfaceInfo[defaultIndex].DNS) + nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.interfaceInfo[ifIndex].DNS) if err != nil { err = plugin.Errorf("Failed to getDNSSettings: %v", err) return nwInfo, err @@ -663,7 +671,7 @@ func (plugin *NetPlugin) createNetworkInternal( IsIPv6Enabled: ipamAddResult.ipv6Enabled, } - if err = addSubnetToNetworkInfo(ipamAddResult, &nwInfo, defaultIndex); err != nil { + if err = addSubnetToNetworkInfo(ipamAddResult, &nwInfo, ifIndex); err != nil { logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) return nwInfo, err @@ -679,8 +687,8 @@ func (plugin *NetPlugin) createNetworkInternal( } // construct network info with ipv4/ipv6 subnets -func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.NetworkInfo, defaultIndex int) error { - for _, ipConfig := range ipamAddResult.interfaceInfo[defaultIndex].IPConfigs { +func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.NetworkInfo, ifIndex int) error { + for _, ipConfig := range ipamAddResult.interfaceInfo[ifIndex].IPConfigs { ip, podSubnetPrefix, err := net.ParseCIDR(ipConfig.Address.String()) if err != nil { return fmt.Errorf("Failed to ParseCIDR for pod subnet prefix: %w", err) @@ -717,10 +725,10 @@ type createEndpointInternalOpt struct { natInfo []policy.NATInfo } -func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, defaultIndex int) (network.EndpointInfo, error) { +func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, ifIndex int) (network.EndpointInfo, error) { epInfo := network.EndpointInfo{} - ifInfo := opt.ipamAddResult.interfaceInfo[defaultIndex] // can also call function, but this will be faster + ifInfo := opt.ipamAddResult.interfaceInfo[ifIndex] epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, ifInfo.DNS, opt.k8sNamespace) if err != nil { err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) @@ -768,7 +776,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, ServiceCidrs: opt.nwCfg.ServiceCidrs, NATInfo: opt.natInfo, NICType: cns.InfraNIC, - SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[defaultIndex].SkipDefaultRoutes, + SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[ifIndex].SkipDefaultRoutes, Routes: ifInfo.Routes, } @@ -806,13 +814,14 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, } epInfos := []*network.EndpointInfo{&epInfo} + epIndex := 0 // epInfo index for InfraNIC // get secondary interface info - for _, secondaryCniResult := range opt.ipamAddResult.interfaceInfo { - switch secondaryCniResult.NICType { + for i := 0; i < len(opt.ipamAddResult.interfaceInfo); i++ { + switch opt.ipamAddResult.interfaceInfo[i].NICType { case cns.DelegatedVMNIC: // secondary var addresses []net.IPNet - for _, ipconfig := range secondaryCniResult.IPConfigs { + for _, ipconfig := range opt.ipamAddResult.interfaceInfo[i].IPConfigs { addresses = append(addresses, ipconfig.Address) } @@ -821,23 +830,25 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, ContainerID: epInfo.ContainerID, NetNsPath: epInfo.NetNsPath, IPAddresses: addresses, - Routes: secondaryCniResult.Routes, - MacAddress: secondaryCniResult.MacAddress, - NICType: secondaryCniResult.NICType, - SkipDefaultRoutes: secondaryCniResult.SkipDefaultRoutes, + Routes: opt.ipamAddResult.interfaceInfo[i].Routes, + MacAddress: opt.ipamAddResult.interfaceInfo[i].MacAddress, + NICType: opt.ipamAddResult.interfaceInfo[i].NICType, + SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[i].SkipDefaultRoutes, }) case cns.BackendNIC: // todo - default: - // InfraNic + case cns.InfraNIC: + epIndex = i continue + default: + // Error catch for unsupported NICType? } } // Create the endpoint. logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) - err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, epInfos, defaultIndex) + err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, epInfos, epIndex) if err != nil { err = plugin.Errorf("Failed to create endpoint: %v", err) } @@ -1428,14 +1439,12 @@ func convertCniResultToInterfaceInfo(result *cniTypesCurr.Result) network.Interf return interfaceInfo } -func findDefaultInterface(ipamAddResult IPAMAddResult) int { - // defaultIf := network.InterfaceInfo{} - for i, ifInfo := range ipamAddResult.interfaceInfo { - if ifInfo.NICType == cns.InfraNIC { - return i +func findDefaultInterface(ipamAddResult IPAMAddResult) (int, error) { + for i := 0; i < len(ipamAddResult.interfaceInfo); i++ { + if ipamAddResult.interfaceInfo[i].NICType == cns.InfraNIC { + return i, nil } } - // error or nil here - return -1 + return -1, errors.New("no NIC was of type InfraNIC") } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 39b496ba3a..fb3d4d50e1 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -158,9 +158,14 @@ func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResu // This will happen during ADD call if ipamAddResult != nil && ipamAddResult.ncResponse != nil { // find defaultInterface within AddResult - defaultIndex := findDefaultInterface(ipamAddResult) + ifIndex, err := findDefaultInterface(*ipamAddResult) + if err != nil { + logger.Error("Error finding InfraNIC interface", + zap.Error(err)) + return "", errors.Wrap(err, "cns did not return an InfraNIC") + } // networkName will look like ~ azure-vlan1-172-28-1-0_24 - ipAddrNet := ipamAddResult.interfaceInfo[defaultIndex].IPConfigs[0].Address + ipAddrNet := ipamAddResult.interfaceInfo[ifIndex].IPConfigs[0].Address prefix, err := netip.ParsePrefix(ipAddrNet.String()) if err != nil { logger.Error("Error parsing network CIDR", diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 94aa3094e8..26bc34b793 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -367,11 +367,13 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, }, interfaceInfo: []network.InterfaceInfo{ - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.240.0.5"), - Mask: net.CIDRMask(24, 32), + { + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("10.240.0.5"), + Mask: net.CIDRMask(24, 32), + }, }, }, }, @@ -402,11 +404,13 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, }, interfaceInfo: []network.InterfaceInfo{ - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP(""), - Mask: net.CIDRMask(24, 32), + { + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP(""), + Mask: net.CIDRMask(24, 32), + }, }, }, }, @@ -437,11 +441,13 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, }, interfaceInfo: []network.InterfaceInfo{ - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.0.00.6"), - Mask: net.CIDRMask(24, 32), + { + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("10.0.00.6"), + Mask: net.CIDRMask(24, 32), + }, }, }, }, @@ -472,11 +478,13 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, }, interfaceInfo: []network.InterfaceInfo{ - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.0.0.6"), - Mask: net.CIDRMask(24, 32), + { + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("10.0.0.6"), + Mask: net.CIDRMask(24, 32), + }, }, }, }, @@ -503,11 +511,13 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ipamAddResult: &IPAMAddResult{ ncResponse: &cns.GetNetworkContainerResponse{}, interfaceInfo: []network.InterfaceInfo{ - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.0.0.6"), - Mask: net.CIDRMask(24, 32), + { + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("10.0.0.6"), + Mask: net.CIDRMask(24, 32), + }, }, }, }, diff --git a/cnm/network/network.go b/cnm/network/network.go index 4358d1a485..078edf76d4 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -247,7 +247,8 @@ func (plugin *netPlugin) createEndpoint(w http.ResponseWriter, r *http.Request) if err != nil { log.Errorf("failed to init CNS client", err) } - err = plugin.nm.CreateEndpoint(cnscli, req.NetworkID, []*network.EndpointInfo{&epInfo}) + // We only pass a single epInfo, so can pass default index as 0 + err = plugin.nm.CreateEndpoint(cnscli, req.NetworkID, []*network.EndpointInfo{&epInfo}, 0) if err != nil { plugin.SendErrorResponse(w, err) return diff --git a/network/endpoint.go b/network/endpoint.go index 2f0d51afca..c17dadb0c6 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -143,21 +143,20 @@ func (nw *network) newEndpoint( nsc NamespaceClientInterface, iptc ipTablesClient, epInfo []*EndpointInfo, - defaultIndex int, + epIndex int, ) (*endpoint, error) { var ep *endpoint var err error defer func() { if err != nil { - // this log is hardcoded for default - logger.Error("Failed to create endpoint with err", zap.String("id", epInfo[defaultIndex].Id), zap.Error(err)) + logger.Error("Failed to create endpoint with err", zap.String("id", epInfo[epIndex].Id), zap.Error(err)) } }() // Call the platform implementation. // Pass nil for epClient and will be initialized in newendpointImpl - ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, nsc, iptc, epInfo, defaultIndex) + ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, nsc, iptc, epInfo, epIndex) if err != nil { return nil, err } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index fcdb5379d8..ce0e2e6e7e 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -58,7 +58,7 @@ func (nw *network) newEndpointImpl( nsc NamespaceClientInterface, iptc ipTablesClient, epInfo []*EndpointInfo, - _ int, + epIndex int, ) (*endpoint, error) { var ( err error @@ -66,7 +66,7 @@ func (nw *network) newEndpointImpl( contIfName string localIP string vlanid = 0 - defaultEpInfo = epInfo[0] // this needs to be changed, out of scope... + defaultEpInfo = epInfo[epIndex] containerIf *net.Interface ) diff --git a/network/endpoint_test.go b/network/endpoint_test.go index 0fc0cf829e..b7e0d8fe47 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -184,7 +184,7 @@ var _ = Describe("Test Endpoint", func() { It("Should be added", func() { // Add endpoint with valid id ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -196,7 +196,7 @@ var _ = Describe("Test Endpoint", func() { extIf: &externalInterface{IPv4Gateway: net.ParseIP("192.168.0.1")}, } ep, err := nw2.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -212,7 +212,7 @@ var _ = Describe("Test Endpoint", func() { Expect(err).ToNot(HaveOccurred()) // Adding endpoint with same id should fail and delete should cleanup the state ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) Expect(err).To(HaveOccurred()) Expect(ep2).To(BeNil()) assert.Contains(GinkgoT(), err.Error(), "Endpoint already exists") @@ -222,7 +222,7 @@ var _ = Describe("Test Endpoint", func() { // Adding an endpoint with an id. mockCli := NewMockEndpointClient(nil) ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) Expect(err).ToNot(HaveOccurred()) Expect(ep2).ToNot(BeNil()) Expect(len(mockCli.endpoints)).To(Equal(1)) @@ -253,11 +253,11 @@ var _ = Describe("Test Endpoint", func() { } return nil - }), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) + }), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) Expect(err).To(HaveOccurred()) Expect(ep).To(BeNil()) ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -283,14 +283,14 @@ var _ = Describe("Test Endpoint", func() { It("Should not endpoint to the network when there is an error", func() { secondaryEpInfo.MacAddress = netio.BadHwAddr // mock netlink will fail to set link state on bad eth ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("SecondaryEndpointClient Error: " + netlink.ErrorMockNetlink.Error())) Expect(ep).To(BeNil()) secondaryEpInfo.MacAddress = netio.HwAddr ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) Expect(err).ToNot(HaveOccurred()) Expect(ep.Id).To(Equal(epInfo.Id)) }) @@ -298,7 +298,7 @@ var _ = Describe("Test Endpoint", func() { It("Should add endpoint when there are no errors", func() { secondaryEpInfo.MacAddress = netio.HwAddr ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) Expect(err).ToNot(HaveOccurred()) Expect(ep.Id).To(Equal(epInfo.Id)) }) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 8aee0f867a..823c7ad737 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -74,18 +74,18 @@ func (nw *network) newEndpointImpl( _ NamespaceClientInterface, _ ipTablesClient, epInfo []*EndpointInfo, - defaultIndex int, + epIndex int, ) (*endpoint, error) { // there is only 1 epInfo for windows, multiple interfaces will be added in the future - if useHnsV2, err := UseHnsV2(epInfo[defaultIndex].NetNsPath); useHnsV2 { + if useHnsV2, err := UseHnsV2(epInfo[epIndex].NetNsPath); useHnsV2 { if err != nil { return nil, err } - return nw.newEndpointImplHnsV2(cli, epInfo[defaultIndex]) + return nw.newEndpointImplHnsV2(cli, epInfo[epIndex]) } - return nw.newEndpointImplHnsV1(epInfo[defaultIndex], plc) + return nw.newEndpointImplHnsV1(epInfo[epIndex], plc) } // newEndpointImplHnsV1 creates a new endpoint in the network using HnsV1 diff --git a/network/manager.go b/network/manager.go index adf98c4ed0..303e812ea9 100644 --- a/network/manager.go +++ b/network/manager.go @@ -98,7 +98,7 @@ type NetworkManager interface { FindNetworkIDFromNetNs(netNs string) (string, error) GetNumEndpointsByContainerID(containerID string) int - CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo, defaultIndex int) error + CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo, epIndex int) error DeleteEndpoint(networkID string, endpointID string, epInfo *EndpointInfo) error GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) @@ -372,7 +372,7 @@ func (nm *networkManager) GetNetworkInfo(networkId string) (NetworkInfo, error) } // CreateEndpoint creates a new container endpoint. -func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo []*EndpointInfo, defaultIndex int) error { +func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo []*EndpointInfo, epIndex int) error { nm.Lock() defer nm.Unlock() @@ -382,14 +382,13 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } if nw.VlanId != 0 { - // leverage passing default index, if failed, call function to find default - if epInfo[defaultIndex].Data[VlanIDKey] == nil { + if epInfo[epIndex].Data[VlanIDKey] == nil { logger.Info("overriding endpoint vlanid with network vlanid") - epInfo[defaultIndex].Data[VlanIDKey] = nw.VlanId + epInfo[epIndex].Data[VlanIDKey] = nw.VlanId } } - ep, err := nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, nm.iptablesClient, epInfo, defaultIndex) + ep, err := nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, nm.iptablesClient, epInfo, epIndex) if err != nil { return err } diff --git a/network/manager_mock.go b/network/manager_mock.go index d490799122..4d4675debe 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -59,7 +59,7 @@ func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfos [] return err } } - // Look into changing this... + nm.TestEndpointInfoMap[epInfos[0].Id] = epInfos[0] return nil } From 59fc47e6490f883a68dada62c4677c3b330eefd8 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 26 Mar 2024 22:03:13 -0700 Subject: [PATCH 011/102] merge nw info fields to ep info and draft new createEndpoint function --- cni/network/network.go | 134 +++++++++++++++++++++++++++++++++++++++++ network/endpoint.go | 19 ++++++ 2 files changed, 153 insertions(+) diff --git a/cni/network/network.go b/cni/network/network.go index 7ac66caab6..cc3c2b9353 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -476,6 +476,8 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } // iterate ipamAddResults and program the endpoint + // GOAL: We should have a populated ipamAddResult by this point so we can do the following: + // for _, ifInfo := range ipamAddResult.interfaceInfo { for i := 0; i < len(ipamAddResults); i++ { var networkID string ipamAddResult = ipamAddResults[i] @@ -603,6 +605,138 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { return nil } +// loop over each interface info and call createEndpoint with it +// you don't have access to a list of interface infos here +// no need for default index because we only trigger the default index code when the type is the infra nic +/** +Overview: +Create endpoint info +Pass pointer to endpoint info to a new function to populate endpoint info fields (right now we don't separate the populating code into functions) +Pass pointer to endpoint info to another new function to populate network info fields (or we can just do it in one big switch statement) + To populate network info, use Paul's code which should get the network info fields from various things passed in (we should pass those things in here too) + Potentially problematic getting the network id because plugin.getNetworkName seems to just always return nwCfg.Name (NEED ASSISTANCE) + But if we do get the network id correctly, just pass to Paul's code to get the network info fields which we can populate epInfo with +After both sections are done, we can call createNetworkInternal (pass in epInfo) and createEndpointInternal (pass in epInfo) + Will need to modify those two functions as code from createEndpointInternal, for example, was pulled out into this function below +*/ +// maybe create a createEndpointOpt struct for all the things we need to pass in +func (plugin *NetPlugin) createEndpoint(opt *createEndpointInternalOpt, ifInfo *network.InterfaceInfo) error { // you can modify to pass in whatever else you need + var ( + epInfo network.EndpointInfo + ) + // taken from createEndpointInternal function + // populate endpoint info fields code is below + // did not move cns client code because we don't need it here + switch ifInfo.NICType { + case cns.DelegatedVMNIC: + // secondary + var addresses []net.IPNet + for _, ipconfig := range ifInfo.IPConfigs { + addresses = append(addresses, ipconfig.Address) + } + + epInfo = network.EndpointInfo{ + ContainerID: epInfo.ContainerID, + NetNsPath: epInfo.NetNsPath, + IPAddresses: addresses, + Routes: ifInfo.Routes, + MacAddress: ifInfo.MacAddress, + NICType: ifInfo.NICType, + SkipDefaultRoutes: ifInfo.SkipDefaultRoutes, + } + case cns.BackendNIC: + // todo + default: + // InfraNic + // taken from create endpoint internal function + // TODO: does this change the behavior (we moved the infra nic code into here rather than above the switch statement)? (it shouldn't) + // at this point you are 100% certain that the interface passed in is the infra nic, and thus, the default interface info + defaultInterfaceInfo := ifInfo + epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultInterfaceInfo.DNS, opt.k8sNamespace) + if err != nil { + err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) + return err + } + policyArgs := PolicyArgs{ + nwInfo: opt.nwInfo, + nwCfg: opt.nwCfg, + ipconfigs: defaultInterfaceInfo.IPConfigs, + } + endpointPolicies, err := getEndpointPolicies(policyArgs) + if err != nil { + logger.Error("Failed to get endpoint policies", zap.Error(err)) + return err + } + + opt.policies = append(opt.policies, endpointPolicies...) + + vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) + if opt.nwCfg.Mode != OpModeTransparent { + // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. + // IT will result in unpredictable behavior if API server decides to + // reorder DELETE and ADD call for new incarnation of same POD. + vethName = fmt.Sprintf("%s%s%s", opt.nwInfo.Id, opt.args.ContainerID, opt.args.IfName) + } + + epInfo = network.EndpointInfo{ + Id: opt.endpointID, + ContainerID: opt.args.ContainerID, + NetNsPath: opt.args.Netns, + IfName: opt.args.IfName, + Data: make(map[string]interface{}), + DNS: epDNSInfo, + Policies: opt.policies, + IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, + EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, + EnableMultiTenancy: opt.nwCfg.MultiTenancy, + EnableInfraVnet: opt.enableInfraVnet, + EnableSnatForDns: opt.enableSnatForDNS, + PODName: opt.k8sPodName, + PODNameSpace: opt.k8sNamespace, + SkipHotAttachEp: false, // Hot attach at the time of endpoint creation + IPV6Mode: opt.nwCfg.IPV6Mode, + VnetCidrs: opt.nwCfg.VnetCidrs, + ServiceCidrs: opt.nwCfg.ServiceCidrs, + NATInfo: opt.natInfo, + NICType: cns.InfraNIC, + SkipDefaultRoutes: ifInfo.SkipDefaultRoutes, // we know we are the default interface at this point + Routes: defaultInterfaceInfo.Routes, + } + + epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) + if err != nil { + logger.Error("failed to get policies from runtime configurations", zap.Error(err)) + return plugin.Errorf(err.Error()) + } + epInfo.Policies = append(epInfo.Policies, epPolicies...) + + // Populate addresses. + for _, ipconfig := range defaultInterfaceInfo.IPConfigs { + epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) + } + + if opt.ipamAddResult.ipv6Enabled { + epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working + } + + if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { + epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address + } + + if opt.nwCfg.MultiTenancy { + plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, defaultInterfaceInfo) + } + + setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) + } + // now need to populate the network info section of the ep info struct here (or you can do it in the switch statement above) + // TODO: add network info populating code + // now our ep info should have the full combined information from both the network and endpoint structs + // call create network internal (need to change function signature) + // call create endpoint internal (need to change function signature) + return nil +} + // cleanup allocated ipv4 and ipv6 addresses if they exist func (plugin *NetPlugin) cleanupAllocationOnError( result []*network.IPConfig, diff --git a/network/endpoint.go b/network/endpoint.go index c17dadb0c6..fdfd243f15 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -93,6 +93,25 @@ type EndpointInfo struct { SkipDefaultRoutes bool HNSEndpointID string HostIfName string + + MasterIfName string // related to HostIfName? + AdapterName string + NetworkId string // referred to as Id in NetworkInfo + Mode string + Subnets []SubnetInfo + PodSubnet SubnetInfo + // DNS omitted + // Policies omitted + BridgeName string + // EnableSnatOnHost omitted + NetNs string // related to NetNsPath? + Options map[string]interface{} // related to Data? + DisableHairpinOnHostInterface bool + // IPV6Mode omitted + IPAMType string + // ServiceCidrs omitted + IsIPv6Enabled bool + // NICType omitted } // RouteInfo contains information about an IP route. From 393b48117c1c8d31e9cddeac8e787f0e649929c2 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 27 Mar 2024 17:27:03 -0400 Subject: [PATCH 012/102] restruct ipamAddResult struct --- cni/network/invoker.go | 5 +- cni/network/invoker_azure.go | 2 +- cni/network/invoker_cns.go | 2 +- cni/network/invoker_cns_test.go | 4 +- cni/network/multitenancy.go | 31 +++-- cni/network/multitenancy_mock.go | 24 ++-- cni/network/multitenancy_test.go | 8 +- cni/network/network.go | 228 +++++++++++++++---------------- network/endpoint.go | 2 + 9 files changed, 158 insertions(+), 148 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 8a47ebc02f..d69817c6bd 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -4,7 +4,6 @@ import ( "net" "github.com/Azure/azure-container-networking/cni" - "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/network" cniSkel "github.com/containernetworking/cni/pkg/skel" ) @@ -30,7 +29,7 @@ type IPAMAddResult struct { // Splitting defaultInterfaceInfo from secondaryInterfacesInfo so we don't need to loop for default CNI result every time interfaceInfo []network.InterfaceInfo // ncResponse is used for Swift 1.0 multitenancy - ncResponse *cns.GetNetworkContainerResponse - hostSubnetPrefix net.IPNet + // ncResponse *cns.GetNetworkContainerResponse + hostSubnetPrefix net.IPNet // Duplicated field for now ipv6Enabled bool } diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index c61d3ce1b7..c3689032e5 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -70,7 +70,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er return addResult, err } if len(result.IPs) > 0 { - addResult.hostSubnetPrefix = result.IPs[0].Address + addResult.hostSubnetPrefix = result.IPs[0].Address // Duplicated field for hostSubnetPrefix } defer func() { diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 46b86a803d..81a463dc9c 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -436,7 +436,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add return fmt.Errorf("unable to parse hostSubnet: %w", err) } - addResult.hostSubnetPrefix = *hostIPNet + addResult.interfaceInfo[ifIndex].HostSubnetPrefix = *hostIPNet addResult.interfaceInfo[ifIndex].NICType = cns.InfraNIC // This can be removed // set subnet prefix for host vm diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index c6cc1d767c..3268b9bf85 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -140,7 +140,8 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { Gw: getTestOverlayGateway(), }, }, - NICType: cns.InfraNIC, + NICType: cns.InfraNIC, + HostSubnetPrefix: *getCIDRNotationForAddress("10.224.0.0/16"), }, wantErr: false, }, @@ -496,6 +497,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { fmt.Print(errInvalidNIC) require.FailNow("No coverage for this NICType") case cns.InfraNIC: + t.Logf("%s", &ifInfo.HostSubnetPrefix) require.Equalf(tt.wantDefaultResult, ifInfo, "incorrect default response") default: require.FailNow("Unsupported NICType") diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index dcb0f0ae10..27d3805f25 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -43,7 +43,7 @@ type MultitenancyClient interface { nwCfg *cni.NetworkConfig, podName string, podNamespace string, - ifName string) ([]IPAMAddResult, error) + ifName string) (IPAMAddResult, error) Init(cnsclient cnsclient, netioshim netioshim) } @@ -189,7 +189,7 @@ func (m *Multitenancy) SetupRoutingForMultitenancy( // get all network container configuration(s) for given orchestratorContext func (m *Multitenancy) GetAllNetworkContainers( ctx context.Context, nwCfg *cni.NetworkConfig, podName, podNamespace, ifName string, -) ([]IPAMAddResult, error) { +) (IPAMAddResult, error) { var podNameWithoutSuffix string if !nwCfg.EnableExactMatchForPodName { @@ -202,7 +202,7 @@ func (m *Multitenancy) GetAllNetworkContainers( ncResponses, hostSubnetPrefixes, err := m.getNetworkContainersInternal(ctx, podNamespace, podNameWithoutSuffix) if err != nil { - return []IPAMAddResult{}, fmt.Errorf("%w", err) + return IPAMAddResult{}, fmt.Errorf("%w", err) } for i := 0; i < len(ncResponses); i++ { @@ -210,24 +210,29 @@ func (m *Multitenancy) GetAllNetworkContainers( if ncResponses[i].LocalIPConfiguration.IPSubnet.IPAddress == "" { logger.Info("Snat IP is not populated for ncs. Got empty string", zap.Any("response", ncResponses)) - return []IPAMAddResult{}, errSnatIP + return IPAMAddResult{}, errSnatIP } } } - ipamResults := make([]IPAMAddResult, len(ncResponses)) + ipamResult := IPAMAddResult{} + ipamResult.interfaceInfo = []network.InterfaceInfo{} + + // ipamResults := make([]IPAMAddResult, len(ncResponses)) // Can use hard coded 0 as ipamResults is empty and we are creating the first interface for i := 0; i < len(ncResponses); i++ { - ipamResults[i].ncResponse = &ncResponses[i] - ipamResults[i].hostSubnetPrefix = hostSubnetPrefixes[i] - ipconfig, routes := convertToIPConfigAndRouteInfo(ipamResults[i].ncResponse) - ipamResults[i].interfaceInfo = append(ipamResults[i].interfaceInfo, network.InterfaceInfo{}) - ipamResults[i].interfaceInfo[0].IPConfigs = []*network.IPConfig{ipconfig} - ipamResults[i].interfaceInfo[0].Routes = routes - ipamResults[i].interfaceInfo[0].NICType = cns.InfraNIC + ipamResult.interfaceInfo = append(ipamResult.interfaceInfo, network.InterfaceInfo{ + NCResponse: &ncResponses[i], + HostSubnetPrefix: hostSubnetPrefixes[i], + }) + + ipconfig, routes := convertToIPConfigAndRouteInfo(ipamResult.interfaceInfo[i].NCResponse) + ipamResult.interfaceInfo[i].IPConfigs = []*network.IPConfig{ipconfig} + ipamResult.interfaceInfo[i].Routes = routes + ipamResult.interfaceInfo[i].NICType = cns.InfraNIC } - return ipamResults, err + return ipamResult, err } // get all network containers configuration for given orchestratorContext diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index bf4497eb75..db5bf50550 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -88,9 +88,9 @@ func (m *MockMultitenancy) GetAllNetworkContainers( podName string, podNamespace string, ifName string, -) ([]IPAMAddResult, error) { +) (IPAMAddResult, error) { if m.fail { - return nil, errMockMulAdd + return IPAMAddResult{}, errMockMulAdd } var cnsResponses []cns.GetNetworkContainerResponse @@ -154,17 +154,19 @@ func (m *MockMultitenancy) GetAllNetworkContainers( ipNets = append(ipNets, *firstIPnet) cnsResponses = append(cnsResponses, *cnsResponseOne) - ipamResults := make([]IPAMAddResult, len(cnsResponses)) + ipamResult := IPAMAddResult{} // Can use hard coded 0 as ipamResults is empty and we are creating the first interface for i := 0; i < len(cnsResponses); i++ { - ipamResults[i].ncResponse = &cnsResponses[i] - ipamResults[i].hostSubnetPrefix = ipNets[i] - ipconfig, routes := convertToIPConfigAndRouteInfo(ipamResults[i].ncResponse) - ipamResults[i].interfaceInfo = append(ipamResults[i].interfaceInfo, network.InterfaceInfo{}) - ipamResults[i].interfaceInfo[0].IPConfigs = []*network.IPConfig{ipconfig} - ipamResults[i].interfaceInfo[0].Routes = routes - ipamResults[i].interfaceInfo[0].NICType = cns.InfraNIC + ipamResult.interfaceInfo = append(ipamResult.interfaceInfo, network.InterfaceInfo{ + NCResponse: &cnsResponses[i], + }) + + ipconfig, routes := convertToIPConfigAndRouteInfo(ipamResult.interfaceInfo[i].NCResponse) + ipamResult.interfaceInfo[i].IPConfigs = []*network.IPConfig{ipconfig} + ipamResult.interfaceInfo[i].Routes = routes + ipamResult.interfaceInfo[i].NICType = cns.InfraNIC + } - return ipamResults, nil + return ipamResult, nil } diff --git a/cni/network/multitenancy_test.go b/cni/network/multitenancy_test.go index 42ef5f39d6..841bfaec57 100644 --- a/cni/network/multitenancy_test.go +++ b/cni/network/multitenancy_test.go @@ -548,9 +548,9 @@ func TestGetMultiTenancyCNIResult(t *testing.T) { require.Error(err) } require.NoError(err) - require.Exactly(tt.want1, got[0].ncResponse) - require.Exactly(tt.want2, got[1].ncResponse) - require.Exactly(tt.want3, got[0].hostSubnetPrefix) + require.Exactly(tt.want1, got.interfaceInfo[0].NCResponse) + require.Exactly(tt.want2, got.interfaceInfo[1].NCResponse) + require.Exactly(tt.want3, got.interfaceInfo[0].HostSubnetPrefix) // check multiple responses tt.want5 = append(tt.want5, *tt.want1, *tt.want2) @@ -691,7 +691,7 @@ func TestGetMultiTenancyCNIResultUnsupportedAPI(t *testing.T) { t.Fatalf("expected an error %+v but none received", err) } require.NoError(err) - require.Exactly(tt.want, got[0].ncResponse) + require.Exactly(tt.want, got.interfaceInfo[0].NCResponse) }) } } diff --git a/cni/network/network.go b/cni/network/network.go index cc3c2b9353..ad4d70f620 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -452,7 +452,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { return fmt.Errorf("%w", err) } - ipamAddResults, err = plugin.multitenancyClient.GetAllNetworkContainers(context.TODO(), nwCfg, k8sPodName, k8sNamespace, args.IfName) + ipamAddResult, err = plugin.multitenancyClient.GetAllNetworkContainers(context.TODO(), nwCfg, k8sPodName, k8sNamespace, args.IfName) if err != nil { err = fmt.Errorf("GetAllNetworkContainers failed for podname %s namespace %s. error: %w", k8sPodName, k8sNamespace, err) logger.Error("GetAllNetworkContainers failed", @@ -462,147 +462,147 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { return err } - if len(ipamAddResults) > 1 && !plugin.isDualNicFeatureSupported(args.Netns) { + if !plugin.isDualNicFeatureSupported(args.Netns) { errMsg := fmt.Sprintf("received multiple NC results %+v from CNS while dualnic feature is not supported", ipamAddResults) logger.Error("received multiple NC results from CNS while dualnic feature is not supported", zap.Any("results", ipamAddResult)) return plugin.Errorf(errMsg) } - } else { - // TODO: refactor this code for simplification - // Add dummy ipamAddResult nil object for single tenancy mode - // this will be used for: ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig) - ipamAddResults = append(ipamAddResults, ipamAddResult) } + // else { + // // TODO: refactor this code for simplification + // // Add dummy ipamAddResult nil object for single tenancy mode + // // this will be used for: ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig) + // ipamAddResults = append(ipamAddResults, ipamAddResult) + // } // iterate ipamAddResults and program the endpoint // GOAL: We should have a populated ipamAddResult by this point so we can do the following: // for _, ifInfo := range ipamAddResult.interfaceInfo { - for i := 0; i < len(ipamAddResults); i++ { - var networkID string - ipamAddResult = ipamAddResults[i] - ifIndex, _ := findDefaultInterface(ipamAddResult) - - options := make(map[string]any) - networkID, err = plugin.getNetworkName(args.Netns, &ipamAddResult, nwCfg) + // for i := 0; i < len(ipamAddResults); i++ { + var networkID string + ifIndex, _ := findDefaultInterface(ipamAddResult) + + options := make(map[string]any) + networkID, err = plugin.getNetworkName(args.Netns, &ipamAddResult, nwCfg) + + endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) + policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) + + // Check whether the network already exists. + nwInfo, nwInfoErr := plugin.nm.GetNetworkInfo(networkID) + // Handle consecutive ADD calls for infrastructure containers. + // This is a temporary work around for issue #57253 of Kubernetes. + // We can delete this if statement once they fix it. + // Issue link: https://github.com/kubernetes/kubernetes/issues/57253 + + if nwInfoErr == nil { + logger.Info("Found network with subnet", + zap.String("network", networkID), + zap.String("subnet", nwInfo.Subnets[0].Prefix.String())) + nwInfo.IPAMType = nwCfg.IPAM.Type + options = nwInfo.Options + + var resultSecondAdd *cniTypesCurr.Result + resultSecondAdd, err = plugin.handleConsecutiveAdd(args, endpointID, networkID, &nwInfo, nwCfg) + if err != nil { + logger.Error("handleConsecutiveAdd failed", zap.Error(err)) + return err + } - endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) - policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) - - // Check whether the network already exists. - nwInfo, nwInfoErr := plugin.nm.GetNetworkInfo(networkID) - // Handle consecutive ADD calls for infrastructure containers. - // This is a temporary work around for issue #57253 of Kubernetes. - // We can delete this if statement once they fix it. - // Issue link: https://github.com/kubernetes/kubernetes/issues/57253 - - if nwInfoErr == nil { - logger.Info("Found network with subnet", - zap.String("network", networkID), - zap.String("subnet", nwInfo.Subnets[0].Prefix.String())) - nwInfo.IPAMType = nwCfg.IPAM.Type - options = nwInfo.Options - - var resultSecondAdd *cniTypesCurr.Result - resultSecondAdd, err = plugin.handleConsecutiveAdd(args, endpointID, networkID, &nwInfo, nwCfg) + if resultSecondAdd != nil { + ifIndex, err = findDefaultInterface(ipamAddResult) if err != nil { - logger.Error("handleConsecutiveAdd failed", zap.Error(err)) - return err - } - - if resultSecondAdd != nil { - ifIndex, err = findDefaultInterface(ipamAddResult) - if err != nil { - ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{}) - ifIndex = len(ipamAddResult.interfaceInfo) - 1 - } - ipamAddResult.interfaceInfo[ifIndex] = convertCniResultToInterfaceInfo(resultSecondAdd) - return nil + ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{}) + ifIndex = len(ipamAddResult.interfaceInfo) - 1 } + ipamAddResult.interfaceInfo[ifIndex] = convertCniResultToInterfaceInfo(resultSecondAdd) + return nil } + } - // Initialize azureipam/cns ipam - if plugin.ipamInvoker == nil { - switch nwCfg.IPAM.Type { - case network.AzureCNS: - plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) + // Initialize azureipam/cns ipam + if plugin.ipamInvoker == nil { + switch nwCfg.IPAM.Type { + case network.AzureCNS: + plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) - default: - plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) - } + default: + plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) } + } - ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} - if !nwCfg.MultiTenancy { - ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig) - if err != nil { - return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) - } - - ifIndex, err = findDefaultInterface(ipamAddResult) - if err != nil { - logger.Error("Error finding InfraNIC interface", - zap.Error(err)) - return errors.Wrap(err, "error finding InfraNIC interface") - } - sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) + ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} + if !nwCfg.MultiTenancy { + ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig) + if err != nil { + return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } - defer func() { //nolint:gocritic - if err != nil { - // for multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips - if !(nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS) { - plugin.cleanupAllocationOnError(ipamAddResult.interfaceInfo[ifIndex].IPConfigs, nwCfg, args, options) - } - } - }() - - // Create network - if nwInfoErr != nil { - // Network does not exist. - logger.Info("Creating network", zap.String("networkID", networkID)) - sendEvent(plugin, fmt.Sprintf("[cni-net] Creating network %v.", networkID)) - // opts map needs to get passed in here - if nwInfo, err = plugin.createNetworkInternal(networkID, policies, ipamAddConfig, ipamAddResult, ifIndex); err != nil { - logger.Error("Create network failed", zap.Error(err)) - return err - } - logger.Info("Created network", - zap.String("networkId", networkID), - zap.String("subnet", ipamAddResult.hostSubnetPrefix.String())) - sendEvent(plugin, fmt.Sprintf("[cni-net] Created network %v with subnet %v.", networkID, ipamAddResult.hostSubnetPrefix.String())) + ifIndex, err = findDefaultInterface(ipamAddResult) + if err != nil { + logger.Error("Error finding InfraNIC interface", + zap.Error(err)) + return errors.Wrap(err, "error finding InfraNIC interface") } + sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) + } - natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) - - createEndpointInternalOpt := createEndpointInternalOpt{ - nwCfg: nwCfg, - cnsNetworkConfig: ipamAddResult.ncResponse, - ipamAddResult: ipamAddResult, - azIpamResult: azIpamResult, - args: args, - nwInfo: &nwInfo, - policies: policies, - endpointID: endpointID, - k8sPodName: k8sPodName, - k8sNamespace: k8sNamespace, - enableInfraVnet: enableInfraVnet, - enableSnatForDNS: enableSnatForDNS, - natInfo: natInfo, + defer func() { //nolint:gocritic + if err != nil { + // for multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips + if !(nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS) { + plugin.cleanupAllocationOnError(ipamAddResult.interfaceInfo[ifIndex].IPConfigs, nwCfg, args, options) + } } + }() - var epInfo network.EndpointInfo - epInfo, err = plugin.createEndpointInternal(&createEndpointInternalOpt, ifIndex) - if err != nil { - logger.Error("Endpoint creation failed", zap.Error(err)) + // Create network + if nwInfoErr != nil { + // Network does not exist. + logger.Info("Creating network", zap.String("networkID", networkID)) + sendEvent(plugin, fmt.Sprintf("[cni-net] Creating network %v.", networkID)) + // opts map needs to get passed in here + if nwInfo, err = plugin.createNetworkInternal(networkID, policies, ipamAddConfig, ipamAddResult, ifIndex); err != nil { + logger.Error("Create network failed", zap.Error(err)) return err } - - sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", - ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + logger.Info("Created network", + zap.String("networkId", networkID), + zap.String("subnet", ipamAddResult.hostSubnetPrefix.String())) + sendEvent(plugin, fmt.Sprintf("[cni-net] Created network %v with subnet %v.", networkID, ipamAddResult.hostSubnetPrefix.String())) + } + + natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) + + createEndpointInternalOpt := createEndpointInternalOpt{ + nwCfg: nwCfg, + cnsNetworkConfig: ipamAddResult.interfaceInfo[0].NCResponse, // Alex will fix this in the for loop + ipamAddResult: ipamAddResult, + azIpamResult: azIpamResult, + args: args, + nwInfo: &nwInfo, + policies: policies, + endpointID: endpointID, + k8sPodName: k8sPodName, + k8sNamespace: k8sNamespace, + enableInfraVnet: enableInfraVnet, + enableSnatForDNS: enableSnatForDNS, + natInfo: natInfo, + } + + var epInfo network.EndpointInfo + epInfo, err = plugin.createEndpointInternal(&createEndpointInternalOpt, ifIndex) + if err != nil { + logger.Error("Endpoint creation failed", zap.Error(err)) + return err } + sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", + ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + return nil + } // loop over each interface info and call createEndpoint with it @@ -810,7 +810,7 @@ func (plugin *NetPlugin) createNetworkInternal( zap.Error(err)) return nwInfo, err } - setNetworkOptions(ipamAddResult.ncResponse, &nwInfo) + setNetworkOptions(ipamAddResult.interfaceInfo[0].NCResponse, &nwInfo) // Alex will fix this err = plugin.nm.CreateNetwork(&nwInfo) if err != nil { diff --git a/network/endpoint.go b/network/endpoint.go index fdfd243f15..9a89f42d2d 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -135,6 +135,8 @@ type InterfaceInfo struct { DNS DNSInfo NICType cns.NICType SkipDefaultRoutes bool + HostSubnetPrefix net.IPNet // Move this field from ipamAddResult + NCResponse *cns.GetNetworkContainerResponse // TODO: create a new struct and keep fields that are being used } type IPConfig struct { From 68d87bcbaef26f7ee87f108d2476980191b879cc Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 27 Mar 2024 18:33:44 -0700 Subject: [PATCH 013/102] reorder code to create epinfo first, and then create network and ep based on epinfo --- cni/network/network.go | 775 +++++++++++++++++++++++++++-------------- network/network.go | 1 + 2 files changed, 512 insertions(+), 264 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index ad4d70f620..2ad6f960fc 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -205,8 +205,21 @@ func (plugin *NetPlugin) Stop() { logger.Info("Plugin stopped") } -// FindMasterInterface returns the name of the master interface. -func (plugin *NetPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPrefix *net.IPNet) string { +// findInterfaceByMAC returns the name of the master interface +func (plugin *NetPlugin) findInterfaceByMAC(macAddress string) string { + interfaces, _ := net.Interfaces() + for _, iface := range interfaces { + // find master interface by macAddress for Swiftv2 + if iface.HardwareAddr.String() == macAddress { + return iface.Name + } + } + // Failed to find a suitable interface. + return "" +} + +// findMasterInterfaceBySubnet returns the name of the master interface. +func (plugin *NetPlugin) findMasterInterfaceBySubnet(nwCfg *cni.NetworkConfig, subnetPrefix *net.IPNet) string { // An explicit master configuration wins. Explicitly specifying a master is // useful if host has multiple interfaces with addresses in the same subnet. if nwCfg.Master != "" { @@ -557,49 +570,88 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } }() - // Create network - if nwInfoErr != nil { - // Network does not exist. - logger.Info("Creating network", zap.String("networkID", networkID)) - sendEvent(plugin, fmt.Sprintf("[cni-net] Creating network %v.", networkID)) - // opts map needs to get passed in here - if nwInfo, err = plugin.createNetworkInternal(networkID, policies, ipamAddConfig, ipamAddResult, ifIndex); err != nil { - logger.Error("Create network failed", zap.Error(err)) - return err + // TODO: is it possible nwInfo is not populated? YES it seems so, in which case the nwInfo is an empty struct! + // TODO: This will mean when we try to do nwInfo.Id or nwInfo.Subnets in createEpInfo, we'll have problems! + for _, ifInfo := range ipamAddResult.interfaceInfo { + // TODO: hopefully I can get natInfo here? + natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) + createEpInfoOpt := createEpInfoOpt{ + nwCfg: nwCfg, + cnsNetworkConfig: ifInfo.NCResponse, + ipamAddResult: ipamAddResult, + azIpamResult: azIpamResult, + args: args, + nwInfo: &nwInfo, // TODO: this is just like a placeholder right? + policies: policies, + endpointID: endpointID, + k8sPodName: k8sPodName, + k8sNamespace: k8sNamespace, + enableInfraVnet: enableInfraVnet, + enableSnatForDNS: enableSnatForDNS, + natInfo: natInfo, + networkID: networkID, // TODO: is this the right place to get the network ID, and if so, it will never change! + + ifInfo: &ifInfo, + ipamAddConfig: &ipamAddConfig, + ipv6Enabled: ipamAddResult.ipv6Enabled, } - logger.Info("Created network", - zap.String("networkId", networkID), - zap.String("subnet", ipamAddResult.hostSubnetPrefix.String())) - sendEvent(plugin, fmt.Sprintf("[cni-net] Created network %v with subnet %v.", networkID, ipamAddResult.hostSubnetPrefix.String())) - } - - natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) + epInfo, err := plugin.createEpInfo(&createEpInfoOpt) + if err != nil { + return errors.Wrap(err, "failed to populate endpoint info struct") + } + err = plugin.createEndpoint(epInfo, nwCfg.CNSUrl) + if err != nil { + return errors.Wrap(err, "failed to create endpoint") + } + // TODO: should this statement be based on the current iteration instead of the constant ifIndex? + sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", + ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + + } + + // // Create network + // if nwInfoErr != nil { + // // Network does not exist. + // logger.Info("Creating network", zap.String("networkID", networkID)) + // sendEvent(plugin, fmt.Sprintf("[cni-net] Creating network %v.", networkID)) + // // opts map needs to get passed in here + // if nwInfo, err = plugin.createNetworkInternal(networkID, policies, ipamAddConfig, ipamAddResult, ifIndex); err != nil { + // logger.Error("Create network failed", zap.Error(err)) + // return err + // } + // logger.Info("Created network", + // zap.String("networkId", networkID), + // zap.String("subnet", ipamAddResult.hostSubnetPrefix.String())) + // sendEvent(plugin, fmt.Sprintf("[cni-net] Created network %v with subnet %v.", networkID, ipamAddResult.hostSubnetPrefix.String())) + // } - createEndpointInternalOpt := createEndpointInternalOpt{ - nwCfg: nwCfg, - cnsNetworkConfig: ipamAddResult.interfaceInfo[0].NCResponse, // Alex will fix this in the for loop - ipamAddResult: ipamAddResult, - azIpamResult: azIpamResult, - args: args, - nwInfo: &nwInfo, - policies: policies, - endpointID: endpointID, - k8sPodName: k8sPodName, - k8sNamespace: k8sNamespace, - enableInfraVnet: enableInfraVnet, - enableSnatForDNS: enableSnatForDNS, - natInfo: natInfo, - } + // natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) + + // createEndpointInternalOpt := createEndpointInternalOpt{ + // nwCfg: nwCfg, + // cnsNetworkConfig: ipamAddResult.interfaceInfo[0].NCResponse, // Alex will fix this in the for loop + // ipamAddResult: ipamAddResult, + // azIpamResult: azIpamResult, + // args: args, + // nwInfo: &nwInfo, + // policies: policies, + // endpointID: endpointID, + // k8sPodName: k8sPodName, + // k8sNamespace: k8sNamespace, + // enableInfraVnet: enableInfraVnet, + // enableSnatForDNS: enableSnatForDNS, + // natInfo: natInfo, + // } - var epInfo network.EndpointInfo - epInfo, err = plugin.createEndpointInternal(&createEndpointInternalOpt, ifIndex) - if err != nil { - logger.Error("Endpoint creation failed", zap.Error(err)) - return err - } + // var epInfo network.EndpointInfo + // epInfo, err = plugin.createEndpointInternal(&createEndpointInternalOpt, ifIndex) + // if err != nil { + // logger.Error("Endpoint creation failed", zap.Error(err)) + // return err + // } - sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", - ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + // sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", + // ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) return nil @@ -620,18 +672,40 @@ After both sections are done, we can call createNetworkInternal (pass in epInfo) Will need to modify those two functions as code from createEndpointInternal, for example, was pulled out into this function below */ // maybe create a createEndpointOpt struct for all the things we need to pass in -func (plugin *NetPlugin) createEndpoint(opt *createEndpointInternalOpt, ifInfo *network.InterfaceInfo) error { // you can modify to pass in whatever else you need +type createEpInfoOpt struct { + nwCfg *cni.NetworkConfig + cnsNetworkConfig *cns.GetNetworkContainerResponse + ipamAddResult IPAMAddResult + azIpamResult *cniTypesCurr.Result + args *cniSkel.CmdArgs + nwInfo *network.NetworkInfo + policies []policy.Policy + endpointID string + k8sPodName string + k8sNamespace string + enableInfraVnet bool + enableSnatForDNS bool + natInfo []policy.NATInfo + networkID string + + ifInfo *network.InterfaceInfo + ipamAddConfig *IPAMAddConfig + ipv6Enabled bool +} + +func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointInfo, error) { // you can modify to pass in whatever else you need var ( epInfo network.EndpointInfo + nwInfo network.NetworkInfo ) // taken from createEndpointInternal function // populate endpoint info fields code is below // did not move cns client code because we don't need it here - switch ifInfo.NICType { + switch opt.ifInfo.NICType { case cns.DelegatedVMNIC: // secondary var addresses []net.IPNet - for _, ipconfig := range ifInfo.IPConfigs { + for _, ipconfig := range opt.ifInfo.IPConfigs { addresses = append(addresses, ipconfig.Address) } @@ -639,33 +713,34 @@ func (plugin *NetPlugin) createEndpoint(opt *createEndpointInternalOpt, ifInfo * ContainerID: epInfo.ContainerID, NetNsPath: epInfo.NetNsPath, IPAddresses: addresses, - Routes: ifInfo.Routes, - MacAddress: ifInfo.MacAddress, - NICType: ifInfo.NICType, - SkipDefaultRoutes: ifInfo.SkipDefaultRoutes, + Routes: opt.ifInfo.Routes, + MacAddress: opt.ifInfo.MacAddress, + NICType: opt.ifInfo.NICType, + SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, } case cns.BackendNIC: // todo - default: + case cns.InfraNIC: // InfraNic // taken from create endpoint internal function // TODO: does this change the behavior (we moved the infra nic code into here rather than above the switch statement)? (it shouldn't) // at this point you are 100% certain that the interface passed in is the infra nic, and thus, the default interface info - defaultInterfaceInfo := ifInfo + defaultInterfaceInfo := opt.ifInfo epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultInterfaceInfo.DNS, opt.k8sNamespace) if err != nil { err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) - return err + return nil, err } + // TODO: (1/2) Possible solution to needing nwInfo for epInfo: move this code related to policies to after we populate the network info policyArgs := PolicyArgs{ - nwInfo: opt.nwInfo, + nwInfo: opt.nwInfo, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? getEndpointPolicies requires nwInfo.Subnets only (checked) nwCfg: opt.nwCfg, ipconfigs: defaultInterfaceInfo.IPConfigs, } endpointPolicies, err := getEndpointPolicies(policyArgs) if err != nil { logger.Error("Failed to get endpoint policies", zap.Error(err)) - return err + return nil, err } opt.policies = append(opt.policies, endpointPolicies...) @@ -675,7 +750,8 @@ func (plugin *NetPlugin) createEndpoint(opt *createEndpointInternalOpt, ifInfo * // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. // IT will result in unpredictable behavior if API server decides to // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", opt.nwInfo.Id, opt.args.ContainerID, opt.args.IfName) + // TODO: (2/2) Possible solution to needing nwInfo for epInfo: move this code related to setEndpointOptions to after we populate the network info + vethName = fmt.Sprintf("%s%s%s", opt.nwInfo.Id, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) The dummy nwInfo has the Id, right? } epInfo = network.EndpointInfo{ @@ -699,14 +775,14 @@ func (plugin *NetPlugin) createEndpoint(opt *createEndpointInternalOpt, ifInfo * ServiceCidrs: opt.nwCfg.ServiceCidrs, NATInfo: opt.natInfo, NICType: cns.InfraNIC, - SkipDefaultRoutes: ifInfo.SkipDefaultRoutes, // we know we are the default interface at this point + SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, // we know we are the default interface at this point Routes: defaultInterfaceInfo.Routes, } epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) if err != nil { logger.Error("failed to get policies from runtime configurations", zap.Error(err)) - return plugin.Errorf(err.Error()) + return nil, plugin.Errorf(err.Error()) } epInfo.Policies = append(epInfo.Policies, epPolicies...) @@ -730,10 +806,160 @@ func (plugin *NetPlugin) createEndpoint(opt *createEndpointInternalOpt, ifInfo * setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) } // now need to populate the network info section of the ep info struct here (or you can do it in the switch statement above) - // TODO: add network info populating code + switch opt.ifInfo.NICType { + case cns.InfraNIC: + masterIfName := plugin.findMasterInterfaceBySubnet(opt.ipamAddConfig.nwCfg, &opt.ifInfo.HostSubnetPrefix) + if masterIfName == "" { + err := plugin.Errorf("Failed to find the master interface") + return nil, err + } + logger.Info("Found master interface", zap.String("ifname", masterIfName)) + if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { + return nil, err + } + + nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) + if err != nil { + return nil, err + } + + nwInfo = network.NetworkInfo{ + Id: opt.networkID, + Mode: opt.ipamAddConfig.nwCfg.Mode, + MasterIfName: masterIfName, + AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, + BridgeName: opt.ipamAddConfig.nwCfg.Bridge, + EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT TODO: Check what takes precedence + Policies: opt.policies, + NetNs: opt.ipamAddConfig.args.Netns, + Options: opt.ipamAddConfig.options, + DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, + IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // TODO: check if IPV6Mode field can be deprecated // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, + ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + IsIPv6Enabled: opt.ipv6Enabled, + NICType: string(cns.InfraNIC), + } + + if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { + logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) + return nil, err + } + setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) + + case cns.DelegatedVMNIC: + masterIfName := plugin.findInterfaceByMAC(opt.ifInfo.MacAddress.String()) + if masterIfName == "" { + err := plugin.Errorf("Failed to find the master interface") + return nil, err + } + logger.Info("Found master interface", zap.String("ifname", masterIfName)) + if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { + return nil, err + } + + nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) + if err != nil { + return nil, err + } + + nwInfo = network.NetworkInfo{ + Id: "azure-" + opt.ifInfo.MacAddress.String(), + Mode: opt.ipamAddConfig.nwCfg.Mode, + MasterIfName: masterIfName, + AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, + BridgeName: opt.ipamAddConfig.nwCfg.Bridge, + EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: see above + DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT: see above + NetNs: opt.ipamAddConfig.args.Netns, + Options: opt.ipamAddConfig.options, + DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, + ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: see above + NICType: string(cns.DelegatedVMNIC), + } + + if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { + logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) + return nil, err + } + setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) + + } + + // populate endpoint info with network info fields + // TODO: maybe instead of creating a network info, directly modify epInfo, but some helper methods rely on networkinfo... + epInfo.MasterIfName = nwInfo.MasterIfName + epInfo.AdapterName = nwInfo.AdapterName + epInfo.NetworkId = nwInfo.Id + epInfo.Mode = nwInfo.Mode + epInfo.Subnets = nwInfo.Subnets + epInfo.PodSubnet = nwInfo.PodSubnet + epInfo.BridgeName = nwInfo.BridgeName + epInfo.NetNs = nwInfo.NetNs + epInfo.Options = nwInfo.Options + epInfo.DisableHairpinOnHostInterface = nwInfo.DisableHairpinOnHostInterface + epInfo.IPAMType = nwInfo.IPAMType + epInfo.IsIPv6Enabled = nwInfo.IsIPv6Enabled + // now our ep info should have the full combined information from both the network and endpoint structs - // call create network internal (need to change function signature) - // call create endpoint internal (need to change function signature) + return &epInfo, nil +} + +// networkID is opt.nwInfo.ID +// cns url is opt.nwCfg.CNSUrl +// actually creates the network and corresponding endpoint +func (plugin *NetPlugin) createEndpoint(epInfo *network.EndpointInfo, CNSUrl string) error { + // check if network exists + nwInfo, nwGetErr := plugin.nm.GetNetworkInfo(epInfo.NetworkId) + if nwGetErr != nil { + // Create the network if it is not found + // populate network info with fields from ep info + nwInfo = network.NetworkInfo{ + MasterIfName: epInfo.MasterIfName, + AdapterName: epInfo.AdapterName, + Id: epInfo.NetworkId, + Mode: epInfo.Mode, + Subnets: epInfo.Subnets, + PodSubnet: epInfo.PodSubnet, + DNS: epInfo.DNS, + Policies: epInfo.Policies, + BridgeName: epInfo.BridgeName, + EnableSnatOnHost: epInfo.EnableSnatOnHost, + NetNs: epInfo.NetNs, + Options: epInfo.Options, + DisableHairpinOnHostInterface: epInfo.DisableHairpinOnHostInterface, + IPV6Mode: epInfo.IPV6Mode, + IPAMType: epInfo.IPAMType, + ServiceCidrs: epInfo.ServiceCidrs, + IsIPv6Enabled: epInfo.IsIPv6Enabled, + NICType: string(epInfo.NICType), + } + err := plugin.nm.CreateNetwork(&nwInfo) + if err != nil { + err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) + return err // added + } + } + + cnsclient, err := cnscli.New(CNSUrl, defaultRequestTimeout) + if err != nil { + logger.Error("failed to initialized cns client", zap.String("url", CNSUrl), + zap.String("error", err.Error())) + return plugin.Errorf(err.Error()) + } + + // Create the endpoint. + logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) + sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) + // TODO: pass in network id, or get network id from ep info-- it makes more sense if it came from epInfo + // since the network is directly related to the epInfo + // TODO: okay to pass in a slice of epInfo this way and just say epIndex is 0, or is refactoring create endpoint needed? + err = plugin.nm.CreateEndpoint(cnsclient, nwInfo.Id, []*network.EndpointInfo{epInfo}, 0) + if err != nil { + err = plugin.Errorf("Failed to create endpoint: %v", err) + return err //added + } return nil } @@ -753,76 +979,97 @@ func (plugin *NetPlugin) cleanupAllocationOnError( } } -func (plugin *NetPlugin) createNetworkInternal( - networkID string, - policies []policy.Policy, - ipamAddConfig IPAMAddConfig, - ipamAddResult IPAMAddResult, - ifIndex int, -) (network.NetworkInfo, error) { - nwInfo := network.NetworkInfo{} - ipamAddResult.hostSubnetPrefix.IP = ipamAddResult.hostSubnetPrefix.IP.Mask(ipamAddResult.hostSubnetPrefix.Mask) - ipamAddConfig.nwCfg.IPAM.Subnet = ipamAddResult.hostSubnetPrefix.String() - // Find the master interface. - masterIfName := plugin.findMasterInterface(ipamAddConfig.nwCfg, &ipamAddResult.hostSubnetPrefix) - if masterIfName == "" { - err := plugin.Errorf("Failed to find the master interface") - return nwInfo, err - } - logger.Info("Found master interface", zap.String("ifname", masterIfName)) - - // Add the master as an external interface. - err := plugin.nm.AddExternalInterface(masterIfName, ipamAddResult.hostSubnetPrefix.String()) +// Copied from paul's commit +// Add the master as an external interface +func (plugin *NetPlugin) addExternalInterface(masterIfName, hostSubnetPrefix string) error { + err := plugin.nm.AddExternalInterface(masterIfName, hostSubnetPrefix) if err != nil { err = plugin.Errorf("Failed to add external interface: %v", err) - return nwInfo, err + return err } + return nil +} - nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.interfaceInfo[ifIndex].DNS) +func (plugin *NetPlugin) getNetworkDNSSettings(nwCfg *cni.NetworkConfig, dns network.DNSInfo) (network.DNSInfo, error) { + nwDNSInfo, err := getNetworkDNSSettings(nwCfg, dns) if err != nil { err = plugin.Errorf("Failed to getDNSSettings: %v", err) - return nwInfo, err + return network.DNSInfo{}, err } - logger.Info("DNS Info", zap.Any("info", nwDNSInfo)) - - // Create the network. - nwInfo = network.NetworkInfo{ - Id: networkID, - Mode: ipamAddConfig.nwCfg.Mode, - MasterIfName: masterIfName, - AdapterName: ipamAddConfig.nwCfg.AdapterName, - BridgeName: ipamAddConfig.nwCfg.Bridge, - EnableSnatOnHost: ipamAddConfig.nwCfg.EnableSnatOnHost, - DNS: nwDNSInfo, - Policies: policies, - NetNs: ipamAddConfig.args.Netns, - Options: ipamAddConfig.options, - DisableHairpinOnHostInterface: ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - IPV6Mode: ipamAddConfig.nwCfg.IPV6Mode, // TODO: check if IPV6Mode field can be deprecated - IPAMType: ipamAddConfig.nwCfg.IPAM.Type, - ServiceCidrs: ipamAddConfig.nwCfg.ServiceCidrs, - IsIPv6Enabled: ipamAddResult.ipv6Enabled, - } - - if err = addSubnetToNetworkInfo(ipamAddResult, &nwInfo, ifIndex); err != nil { - logger.Info("Failed to add subnets to networkInfo", - zap.Error(err)) - return nwInfo, err - } - setNetworkOptions(ipamAddResult.interfaceInfo[0].NCResponse, &nwInfo) // Alex will fix this - - err = plugin.nm.CreateNetwork(&nwInfo) - if err != nil { - err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) - } - - return nwInfo, err + return nwDNSInfo, nil } +// func (plugin *NetPlugin) createNetworkInternal( +// networkID string, +// policies []policy.Policy, +// ipamAddConfig IPAMAddConfig, +// ipamAddResult IPAMAddResult, +// ifIndex int, +// ) (network.NetworkInfo, error) { +// nwInfo := network.NetworkInfo{} +// ipamAddResult.hostSubnetPrefix.IP = ipamAddResult.hostSubnetPrefix.IP.Mask(ipamAddResult.hostSubnetPrefix.Mask) +// ipamAddConfig.nwCfg.IPAM.Subnet = ipamAddResult.hostSubnetPrefix.String() +// // Find the master interface. +// masterIfName := plugin.findMasterInterface(ipamAddConfig.nwCfg, &ipamAddResult.hostSubnetPrefix) +// if masterIfName == "" { +// err := plugin.Errorf("Failed to find the master interface") +// return nwInfo, err +// } +// logger.Info("Found master interface", zap.String("ifname", masterIfName)) + +// // Add the master as an external interface. +// err := plugin.nm.AddExternalInterface(masterIfName, ipamAddResult.hostSubnetPrefix.String()) +// if err != nil { +// err = plugin.Errorf("Failed to add external interface: %v", err) +// return nwInfo, err +// } + +// nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.interfaceInfo[ifIndex].DNS) +// if err != nil { +// err = plugin.Errorf("Failed to getDNSSettings: %v", err) +// return nwInfo, err +// } + +// logger.Info("DNS Info", zap.Any("info", nwDNSInfo)) + +// // Create the network. +// nwInfo = network.NetworkInfo{ +// Id: networkID, +// Mode: ipamAddConfig.nwCfg.Mode, +// MasterIfName: masterIfName, +// AdapterName: ipamAddConfig.nwCfg.AdapterName, +// BridgeName: ipamAddConfig.nwCfg.Bridge, +// EnableSnatOnHost: ipamAddConfig.nwCfg.EnableSnatOnHost, +// DNS: nwDNSInfo, +// Policies: policies, +// NetNs: ipamAddConfig.args.Netns, +// Options: ipamAddConfig.options, +// DisableHairpinOnHostInterface: ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, +// IPV6Mode: ipamAddConfig.nwCfg.IPV6Mode, // TODO: check if IPV6Mode field can be deprecated +// IPAMType: ipamAddConfig.nwCfg.IPAM.Type, +// ServiceCidrs: ipamAddConfig.nwCfg.ServiceCidrs, +// IsIPv6Enabled: ipamAddResult.ipv6Enabled, +// } + +// if err = addSubnetToNetworkInfo(ipamAddResult, &nwInfo, ifIndex); err != nil { +// logger.Info("Failed to add subnets to networkInfo", +// zap.Error(err)) +// return nwInfo, err +// } +// setNetworkOptions(ipamAddResult.interfaceInfo[0].NCResponse, &nwInfo) // Alex will fix this + +// err = plugin.nm.CreateNetwork(&nwInfo) +// if err != nil { +// err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) +// } + +// return nwInfo, err +// } + // construct network info with ipv4/ipv6 subnets -func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.NetworkInfo, ifIndex int) error { - for _, ipConfig := range ipamAddResult.interfaceInfo[ifIndex].IPConfigs { +func addSubnetToNetworkInfo(interfaceInfo network.InterfaceInfo, nwInfo *network.NetworkInfo) error { + for _, ipConfig := range interfaceInfo.IPConfigs { ip, podSubnetPrefix, err := net.ParseCIDR(ipConfig.Address.String()) if err != nil { return fmt.Errorf("Failed to ParseCIDR for pod subnet prefix: %w", err) @@ -843,152 +1090,152 @@ func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.Network return nil } -type createEndpointInternalOpt struct { - nwCfg *cni.NetworkConfig - cnsNetworkConfig *cns.GetNetworkContainerResponse - ipamAddResult IPAMAddResult - azIpamResult *cniTypesCurr.Result - args *cniSkel.CmdArgs - nwInfo *network.NetworkInfo - policies []policy.Policy - endpointID string - k8sPodName string - k8sNamespace string - enableInfraVnet bool - enableSnatForDNS bool - natInfo []policy.NATInfo -} - -func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, ifIndex int) (network.EndpointInfo, error) { - epInfo := network.EndpointInfo{} - - ifInfo := opt.ipamAddResult.interfaceInfo[ifIndex] - epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, ifInfo.DNS, opt.k8sNamespace) - if err != nil { - err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) - return epInfo, err - } - policyArgs := PolicyArgs{ - nwInfo: opt.nwInfo, - nwCfg: opt.nwCfg, - ipconfigs: ifInfo.IPConfigs, - } - endpointPolicies, err := getEndpointPolicies(policyArgs) - if err != nil { - logger.Error("Failed to get endpoint policies", zap.Error(err)) - return epInfo, err - } - - opt.policies = append(opt.policies, endpointPolicies...) - - vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) - if opt.nwCfg.Mode != OpModeTransparent { - // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. - // IT will result in unpredictable behavior if API server decides to - // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", opt.nwInfo.Id, opt.args.ContainerID, opt.args.IfName) - } - - epInfo = network.EndpointInfo{ - Id: opt.endpointID, - ContainerID: opt.args.ContainerID, - NetNsPath: opt.args.Netns, - IfName: opt.args.IfName, - Data: make(map[string]interface{}), - DNS: epDNSInfo, - Policies: opt.policies, - IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, - EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, - EnableMultiTenancy: opt.nwCfg.MultiTenancy, - EnableInfraVnet: opt.enableInfraVnet, - EnableSnatForDns: opt.enableSnatForDNS, - PODName: opt.k8sPodName, - PODNameSpace: opt.k8sNamespace, - SkipHotAttachEp: false, // Hot attach at the time of endpoint creation - IPV6Mode: opt.nwCfg.IPV6Mode, - VnetCidrs: opt.nwCfg.VnetCidrs, - ServiceCidrs: opt.nwCfg.ServiceCidrs, - NATInfo: opt.natInfo, - NICType: cns.InfraNIC, - SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[ifIndex].SkipDefaultRoutes, - Routes: ifInfo.Routes, - } - - epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) - if err != nil { - logger.Error("failed to get policies from runtime configurations", zap.Error(err)) - return epInfo, plugin.Errorf(err.Error()) - } - epInfo.Policies = append(epInfo.Policies, epPolicies...) - - // Populate addresses. - for _, ipconfig := range ifInfo.IPConfigs { - epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) - } - - if opt.ipamAddResult.ipv6Enabled { - epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working - } - - if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { - epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address - } - - if opt.nwCfg.MultiTenancy { - plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, &ifInfo) - } - - setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) - - cnsclient, err := cnscli.New(opt.nwCfg.CNSUrl, defaultRequestTimeout) - if err != nil { - logger.Error("failed to initialized cns client", zap.String("url", opt.nwCfg.CNSUrl), - zap.String("error", err.Error())) - return epInfo, plugin.Errorf(err.Error()) - } - - epInfos := []*network.EndpointInfo{&epInfo} - epIndex := 0 // epInfo index for InfraNIC - // get secondary interface info - for i := 0; i < len(opt.ipamAddResult.interfaceInfo); i++ { - switch opt.ipamAddResult.interfaceInfo[i].NICType { - case cns.DelegatedVMNIC: - // secondary - var addresses []net.IPNet - for _, ipconfig := range opt.ipamAddResult.interfaceInfo[i].IPConfigs { - addresses = append(addresses, ipconfig.Address) - } - - epInfos = append(epInfos, - &network.EndpointInfo{ - ContainerID: epInfo.ContainerID, - NetNsPath: epInfo.NetNsPath, - IPAddresses: addresses, - Routes: opt.ipamAddResult.interfaceInfo[i].Routes, - MacAddress: opt.ipamAddResult.interfaceInfo[i].MacAddress, - NICType: opt.ipamAddResult.interfaceInfo[i].NICType, - SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[i].SkipDefaultRoutes, - }) - case cns.BackendNIC: - // todo - case cns.InfraNIC: - epIndex = i - continue - default: - // Error catch for unsupported NICType? - } - } - - // Create the endpoint. - logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) - sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) - err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, epInfos, epIndex) - if err != nil { - err = plugin.Errorf("Failed to create endpoint: %v", err) - } - - return epInfo, err -} +// type createEndpointInternalOpt struct { +// nwCfg *cni.NetworkConfig +// cnsNetworkConfig *cns.GetNetworkContainerResponse +// ipamAddResult IPAMAddResult +// azIpamResult *cniTypesCurr.Result +// args *cniSkel.CmdArgs +// nwInfo *network.NetworkInfo +// policies []policy.Policy +// endpointID string +// k8sPodName string +// k8sNamespace string +// enableInfraVnet bool +// enableSnatForDNS bool +// natInfo []policy.NATInfo +// } + +// func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, ifIndex int) (network.EndpointInfo, error) { +// epInfo := network.EndpointInfo{} + +// ifInfo := opt.ipamAddResult.interfaceInfo[ifIndex] +// epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, ifInfo.DNS, opt.k8sNamespace) +// if err != nil { +// err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) +// return epInfo, err +// } +// policyArgs := PolicyArgs{ +// nwInfo: opt.nwInfo, +// nwCfg: opt.nwCfg, +// ipconfigs: ifInfo.IPConfigs, +// } +// endpointPolicies, err := getEndpointPolicies(policyArgs) +// if err != nil { +// logger.Error("Failed to get endpoint policies", zap.Error(err)) +// return epInfo, err +// } + +// opt.policies = append(opt.policies, endpointPolicies...) + +// vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) +// if opt.nwCfg.Mode != OpModeTransparent { +// // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. +// // IT will result in unpredictable behavior if API server decides to +// // reorder DELETE and ADD call for new incarnation of same POD. +// vethName = fmt.Sprintf("%s%s%s", opt.nwInfo.Id, opt.args.ContainerID, opt.args.IfName) +// } + +// epInfo = network.EndpointInfo{ +// Id: opt.endpointID, +// ContainerID: opt.args.ContainerID, +// NetNsPath: opt.args.Netns, +// IfName: opt.args.IfName, +// Data: make(map[string]interface{}), +// DNS: epDNSInfo, +// Policies: opt.policies, +// IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, +// EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, +// EnableMultiTenancy: opt.nwCfg.MultiTenancy, +// EnableInfraVnet: opt.enableInfraVnet, +// EnableSnatForDns: opt.enableSnatForDNS, +// PODName: opt.k8sPodName, +// PODNameSpace: opt.k8sNamespace, +// SkipHotAttachEp: false, // Hot attach at the time of endpoint creation +// IPV6Mode: opt.nwCfg.IPV6Mode, +// VnetCidrs: opt.nwCfg.VnetCidrs, +// ServiceCidrs: opt.nwCfg.ServiceCidrs, +// NATInfo: opt.natInfo, +// NICType: cns.InfraNIC, +// SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[ifIndex].SkipDefaultRoutes, +// Routes: ifInfo.Routes, +// } + +// epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) +// if err != nil { +// logger.Error("failed to get policies from runtime configurations", zap.Error(err)) +// return epInfo, plugin.Errorf(err.Error()) +// } +// epInfo.Policies = append(epInfo.Policies, epPolicies...) + +// // Populate addresses. +// for _, ipconfig := range ifInfo.IPConfigs { +// epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) +// } + +// if opt.ipamAddResult.ipv6Enabled { +// epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working +// } + +// if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { +// epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address +// } + +// if opt.nwCfg.MultiTenancy { +// plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, &ifInfo) +// } + +// setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) + +// cnsclient, err := cnscli.New(opt.nwCfg.CNSUrl, defaultRequestTimeout) +// if err != nil { +// logger.Error("failed to initialized cns client", zap.String("url", opt.nwCfg.CNSUrl), +// zap.String("error", err.Error())) +// return epInfo, plugin.Errorf(err.Error()) +// } + +// epInfos := []*network.EndpointInfo{&epInfo} +// epIndex := 0 // epInfo index for InfraNIC +// // get secondary interface info +// for i := 0; i < len(opt.ipamAddResult.interfaceInfo); i++ { +// switch opt.ipamAddResult.interfaceInfo[i].NICType { +// case cns.DelegatedVMNIC: +// // secondary +// var addresses []net.IPNet +// for _, ipconfig := range opt.ipamAddResult.interfaceInfo[i].IPConfigs { +// addresses = append(addresses, ipconfig.Address) +// } + +// epInfos = append(epInfos, +// &network.EndpointInfo{ +// ContainerID: epInfo.ContainerID, +// NetNsPath: epInfo.NetNsPath, +// IPAddresses: addresses, +// Routes: opt.ipamAddResult.interfaceInfo[i].Routes, +// MacAddress: opt.ipamAddResult.interfaceInfo[i].MacAddress, +// NICType: opt.ipamAddResult.interfaceInfo[i].NICType, +// SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[i].SkipDefaultRoutes, +// }) +// case cns.BackendNIC: +// // todo +// case cns.InfraNIC: +// epIndex = i +// continue +// default: +// // Error catch for unsupported NICType? +// } +// } + +// // Create the endpoint. +// logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) +// sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) +// err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, epInfos, epIndex) +// if err != nil { +// err = plugin.Errorf("Failed to create endpoint: %v", err) +// } + +// return epInfo, err +// } // Get handles CNI Get commands. func (plugin *NetPlugin) Get(args *cniSkel.CmdArgs) error { diff --git a/network/network.go b/network/network.go index 0c0622cec0..7caab89fce 100644 --- a/network/network.go +++ b/network/network.go @@ -75,6 +75,7 @@ type NetworkInfo struct { IPAMType string ServiceCidrs string IsIPv6Enabled bool + NICType string } // SubnetInfo contains subnet information for a container network. From 157ee72b148398cfac2527fad78673d3cf1b879e Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 28 Mar 2024 10:59:56 -0400 Subject: [PATCH 014/102] add getNwInfo and generate ipamAddResult --- cni/network/invoker.go | 5 +- cni/network/invoker_azure.go | 8 ++- cni/network/invoker_mock.go | 5 +- cni/network/network.go | 130 ++++++++++++++++++----------------- 4 files changed, 76 insertions(+), 72 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index d69817c6bd..4decc1c1e3 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -28,8 +28,5 @@ type IPAMAddConfig struct { type IPAMAddResult struct { // Splitting defaultInterfaceInfo from secondaryInterfacesInfo so we don't need to loop for default CNI result every time interfaceInfo []network.InterfaceInfo - // ncResponse is used for Swift 1.0 multitenancy - // ncResponse *cns.GetNetworkContainerResponse - hostSubnetPrefix net.IPNet // Duplicated field for now - ipv6Enabled bool + ipv6Enabled bool } diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index c3689032e5..4c3ca88103 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -69,8 +69,14 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er err = invoker.plugin.Errorf("Failed to allocate pool: %v", err) return addResult, err } + + addResult.interfaceInfo = []network.InterfaceInfo{} + addResult.interfaceInfo = append(addResult.interfaceInfo, network.InterfaceInfo{ + HostSubnetPrefix: net.IPNet{}, + }) + if len(result.IPs) > 0 { - addResult.hostSubnetPrefix = result.IPs[0].Address // Duplicated field for hostSubnetPrefix + addResult.interfaceInfo[0].HostSubnetPrefix = result.IPs[0].Address } defer func() { diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 4946ddd39d..ee303d8eff 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -52,8 +52,6 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes return ipamAddResult, errV4 } - ipamAddResult.hostSubnetPrefix = net.IPNet{} - ipv4Str := "10.240.0.5" if _, ok := invoker.ipMap["10.240.0.5/24"]; ok { ipv4Str = "10.240.0.6" @@ -66,7 +64,8 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes IPConfigs: []*network.IPConfig{ {Address: ipnet, Gateway: gwIP}, }, - NICType: cns.InfraNIC, + NICType: cns.InfraNIC, + HostSubnetPrefix: net.IPNet{}, }) invoker.ipMap[ipnet.String()] = true if invoker.v6Fail { diff --git a/cni/network/network.go b/cni/network/network.go index 2ad6f960fc..18b878c702 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -326,6 +326,22 @@ func addNatIPV6SubnetInfo(nwCfg *cni.NetworkConfig, } } +func (plugin *NetPlugin) addIpamInvoker(ipamAddConfig IPAMAddConfig) (IPAMAddResult, error) { + ipamAddResult, err := plugin.ipamInvoker.Add(ipamAddConfig) + if err != nil { + return IPAMAddResult{}, err + } + sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam interfaces: %+v", ipamAddResult.interfaceInfo)) + return ipamAddResult, nil +} + +// get network info +func (plugin *NetPlugin) getNetworkInfo(netNs string, ipamAddResult IPAMAddResult, nwCfg *cni.NetworkConfig) (network.NetworkInfo, string, error) { + networkID, _ := plugin.getNetworkName(netNs, &ipamAddResult, nwCfg) + nwInfo, nwError := plugin.nm.GetNetworkInfo(networkID) + return nwInfo, networkID, nwError +} + // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -375,31 +391,26 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { telemetry.SendCNIMetric(&cniMetric, plugin.tb) // Add Interfaces to result. - ifIndex, _ := findDefaultInterface(ipamAddResult) - if ifIndex < 0 { - ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{}) - ifIndex = len(ipamAddResult.interfaceInfo) - 1 - } - defaultCniResult := convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[ifIndex], args.IfName) - - addSnatInterface(nwCfg, defaultCniResult) + for _, intfInfo := range ipamAddResult.interfaceInfo { + cniResult := convertInterfaceInfoToCniResult(intfInfo, args.IfName) + addSnatInterface(nwCfg, cniResult) + // Convert result to the requested CNI version. + res, vererr := cniResult.GetAsVersion(nwCfg.CNIVersion) + if vererr != nil { + logger.Error("GetAsVersion failed", zap.Error(vererr)) + plugin.Error(vererr) + } - // Convert result to the requested CNI version. - res, vererr := defaultCniResult.GetAsVersion(nwCfg.CNIVersion) - if vererr != nil { - logger.Error("GetAsVersion failed", zap.Error(vererr)) - plugin.Error(vererr) - } + if err == nil && res != nil { + // Output the result to stdout. + res.Print() + } - if err == nil && res != nil { - // Output the result to stdout. - res.Print() + logger.Info("ADD command completed for", + zap.String("pod", k8sPodName), + zap.Any("IPs", cniResult.IPs), + zap.Error(err)) } - - logger.Info("ADD command completed for", - zap.String("pod", k8sPodName), - zap.Any("IPs", defaultCniResult.IPs), - zap.Error(err)) }() // Parse Pod arguments. @@ -482,28 +493,46 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { return plugin.Errorf(errMsg) } } - // else { - // // TODO: refactor this code for simplification - // // Add dummy ipamAddResult nil object for single tenancy mode - // // this will be used for: ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig) - // ipamAddResults = append(ipamAddResults, ipamAddResult) - // } - // iterate ipamAddResults and program the endpoint - // GOAL: We should have a populated ipamAddResult by this point so we can do the following: - // for _, ifInfo := range ipamAddResult.interfaceInfo { - // for i := 0; i < len(ipamAddResults); i++ { - var networkID string - ifIndex, _ := findDefaultInterface(ipamAddResult) + var ( + nwInfo network.NetworkInfo + networkID string + nwInfoErr error + ) - options := make(map[string]any) - networkID, err = plugin.getNetworkName(args.Netns, &ipamAddResult, nwCfg) + ifIndex, _ := findDefaultInterface(ipamAddResult) endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) + options := make(map[string]any) + ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} + + if plugin.ipamInvoker == nil { + switch nwCfg.IPAM.Type { + case network.AzureCNS: + plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) + default: + nwInfo, networkID, nwInfoErr = plugin.getNetworkInfo(args.Netns, ipamAddResult, nwCfg) + plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) + } + } + + if !nwCfg.MultiTenancy { + ipamAddResult, err = plugin.addIpamInvoker(ipamAddConfig) + if err != nil { + return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) + } + if ifIndex == -1 { + ifIndex, _ = findDefaultInterface(ipamAddResult) + } + // This proably needs to be changed as we return all interfaces... + sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) + } + // Check whether the network already exists. - nwInfo, nwInfoErr := plugin.nm.GetNetworkInfo(networkID) + nwInfo, networkID, nwInfoErr = plugin.getNetworkInfo(args.Netns, ipamAddResult, nwCfg) + // Handle consecutive ADD calls for infrastructure containers. // This is a temporary work around for issue #57253 of Kubernetes. // We can delete this if statement once they fix it. @@ -534,33 +563,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } } - // Initialize azureipam/cns ipam - if plugin.ipamInvoker == nil { - switch nwCfg.IPAM.Type { - case network.AzureCNS: - plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) - - default: - plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) - } - } - - ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} - if !nwCfg.MultiTenancy { - ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig) - if err != nil { - return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) - } - - ifIndex, err = findDefaultInterface(ipamAddResult) - if err != nil { - logger.Error("Error finding InfraNIC interface", - zap.Error(err)) - return errors.Wrap(err, "error finding InfraNIC interface") - } - sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) - } - defer func() { //nolint:gocritic if err != nil { // for multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips From 6a5452a99faf0e3e6c84b9b91106b488b16daf9d Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 28 Mar 2024 16:58:02 -0400 Subject: [PATCH 015/102] fix network windows.go --- cni/network/network.go | 50 ++++++++++++++--------------- cni/network/network_windows.go | 6 ++-- cni/network/network_windows_test.go | 42 ++++++++++++------------ 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 18b878c702..a674ebe924 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -466,6 +466,15 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { return fmt.Errorf("failed to create cns client with error: %w", err) } + options := make(map[string]any) + ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} + + var ( + nwInfo network.NetworkInfo + networkID string + nwInfoErr error + ) + if nwCfg.MultiTenancy { plugin.report.Context = "AzureCNIMultitenancy" plugin.multitenancyClient.Init(cnsClient, AzureNetIOShim{}) @@ -492,44 +501,35 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { zap.Any("results", ipamAddResult)) return plugin.Errorf(errMsg) } - } - - var ( - nwInfo network.NetworkInfo - networkID string - nwInfoErr error - ) - - ifIndex, _ := findDefaultInterface(ipamAddResult) - - endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) - policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) - - options := make(map[string]any) - ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} - - if plugin.ipamInvoker == nil { - switch nwCfg.IPAM.Type { - case network.AzureCNS: - plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) - default: - nwInfo, networkID, nwInfoErr = plugin.getNetworkInfo(args.Netns, ipamAddResult, nwCfg) - plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) + } else { + if plugin.ipamInvoker == nil { + switch nwCfg.IPAM.Type { + case network.AzureCNS: + plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) + default: + nwInfo, networkID, nwInfoErr = plugin.getNetworkInfo(args.Netns, ipamAddResult, nwCfg) + plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) + } } - } - if !nwCfg.MultiTenancy { ipamAddResult, err = plugin.addIpamInvoker(ipamAddConfig) if err != nil { return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } + ifIndex, _ := findDefaultInterface(ipamAddResult) + if ifIndex == -1 { ifIndex, _ = findDefaultInterface(ipamAddResult) } // This proably needs to be changed as we return all interfaces... sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) + } + ifIndex, _ := findDefaultInterface(ipamAddResult) + endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) + policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) + // Check whether the network already exists. nwInfo, networkID, nwInfoErr = plugin.getNetworkInfo(args.Netns, ipamAddResult, nwCfg) diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index fb3d4d50e1..1fb2e4936c 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -156,9 +156,9 @@ func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResu // First try to build the network name from the cnsResponse if present // This will happen during ADD call - if ipamAddResult != nil && ipamAddResult.ncResponse != nil { + ifIndex, err := findDefaultInterface(*ipamAddResult) + if ipamAddResult != nil && ipamAddResult.interfaceInfo[ifIndex].NCResponse != nil { // find defaultInterface within AddResult - ifIndex, err := findDefaultInterface(*ipamAddResult) if err != nil { logger.Error("Error finding InfraNIC interface", zap.Error(err)) @@ -175,7 +175,7 @@ func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResu } networkName := strings.ReplaceAll(prefix.Masked().String(), ".", "-") networkName = strings.ReplaceAll(networkName, "/", "_") - networkName = fmt.Sprintf("%s-vlan%v-%v", nwCfg.Name, ipamAddResult.ncResponse.MultiTenancyInfo.ID, networkName) + networkName = fmt.Sprintf("%s-vlan%v-%v", nwCfg.Name, ipamAddResult.interfaceInfo[ifIndex].NCResponse.MultiTenancyInfo.ID, networkName) return networkName, nil } diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 26bc34b793..39a55f83c7 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -361,11 +361,6 @@ func TestGetNetworkNameFromCNS(t *testing.T) { MultiTenancy: true, }, ipamAddResult: &IPAMAddResult{ - ncResponse: &cns.GetNetworkContainerResponse{ - MultiTenancyInfo: cns.MultiTenancyInfo{ - ID: 1, - }, - }, interfaceInfo: []network.InterfaceInfo{ { IPConfigs: []*network.IPConfig{ @@ -376,6 +371,11 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, }, }, + NCResponse: &cns.GetNetworkContainerResponse{ + MultiTenancyInfo: cns.MultiTenancyInfo{ + ID: 1, + }, + }, }, }, }, @@ -398,11 +398,6 @@ func TestGetNetworkNameFromCNS(t *testing.T) { MultiTenancy: true, }, ipamAddResult: &IPAMAddResult{ - ncResponse: &cns.GetNetworkContainerResponse{ - MultiTenancyInfo: cns.MultiTenancyInfo{ - ID: 1, - }, - }, interfaceInfo: []network.InterfaceInfo{ { IPConfigs: []*network.IPConfig{ @@ -413,6 +408,11 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, }, }, + NCResponse: &cns.GetNetworkContainerResponse{ + MultiTenancyInfo: cns.MultiTenancyInfo{ + ID: 1, + }, + }, }, }, }, @@ -435,11 +435,6 @@ func TestGetNetworkNameFromCNS(t *testing.T) { MultiTenancy: true, }, ipamAddResult: &IPAMAddResult{ - ncResponse: &cns.GetNetworkContainerResponse{ - MultiTenancyInfo: cns.MultiTenancyInfo{ - ID: 1, - }, - }, interfaceInfo: []network.InterfaceInfo{ { IPConfigs: []*network.IPConfig{ @@ -450,6 +445,11 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, }, }, + NCResponse: &cns.GetNetworkContainerResponse{ + MultiTenancyInfo: cns.MultiTenancyInfo{ + ID: 1, + }, + }, }, }, }, @@ -472,11 +472,6 @@ func TestGetNetworkNameFromCNS(t *testing.T) { MultiTenancy: true, }, ipamAddResult: &IPAMAddResult{ - ncResponse: &cns.GetNetworkContainerResponse{ - MultiTenancyInfo: cns.MultiTenancyInfo{ - ID: 1, - }, - }, interfaceInfo: []network.InterfaceInfo{ { IPConfigs: []*network.IPConfig{ @@ -487,6 +482,11 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, }, }, + NCResponse: &cns.GetNetworkContainerResponse{ + MultiTenancyInfo: cns.MultiTenancyInfo{ + ID: 1, + }, + }, }, }, }, @@ -509,7 +509,6 @@ func TestGetNetworkNameFromCNS(t *testing.T) { MultiTenancy: false, }, ipamAddResult: &IPAMAddResult{ - ncResponse: &cns.GetNetworkContainerResponse{}, interfaceInfo: []network.InterfaceInfo{ { IPConfigs: []*network.IPConfig{ @@ -520,6 +519,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, }, }, + NCResponse: &cns.GetNetworkContainerResponse{}, }, }, }, From bd1d2541dc62ad0df2571c4fd923ccdda5542129 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 28 Mar 2024 13:58:42 -0700 Subject: [PATCH 016/102] create nw info first and create nw and ep dns info --- cni/network/network.go | 180 +++++++++--------- network/endpoint.go | 5 +- network/endpoint_linux.go | 2 +- network/endpoint_windows.go | 14 +- network/endpoint_windows_test.go | 10 +- network/ovs_endpoint_snatroute_linux.go | 2 +- ...ansparent_vlan_endpoint_snatroute_linux.go | 2 +- 7 files changed, 109 insertions(+), 106 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 18b878c702..9c5eee006e 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -700,6 +700,90 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn epInfo network.EndpointInfo nwInfo network.NetworkInfo ) + + // populate endpoint info section + switch opt.ifInfo.NICType { + case cns.InfraNIC: + masterIfName := plugin.findMasterInterfaceBySubnet(opt.ipamAddConfig.nwCfg, &opt.ifInfo.HostSubnetPrefix) + if masterIfName == "" { + err := plugin.Errorf("Failed to find the master interface") + return nil, err + } + logger.Info("Found master interface", zap.String("ifname", masterIfName)) + if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { + return nil, err + } + + nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) + if err != nil { + return nil, err + } + + nwInfo = network.NetworkInfo{ + Id: opt.networkID, + Mode: opt.ipamAddConfig.nwCfg.Mode, + MasterIfName: masterIfName, + AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, + BridgeName: opt.ipamAddConfig.nwCfg.Bridge, + EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT TODO: Check what takes precedence + Policies: opt.policies, + NetNs: opt.ipamAddConfig.args.Netns, + Options: opt.ipamAddConfig.options, + DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, + IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // TODO: check if IPV6Mode field can be deprecated // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, + ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + IsIPv6Enabled: opt.ipv6Enabled, + NICType: string(cns.InfraNIC), + } + + if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { + logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) + return nil, err + } + setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) + + case cns.DelegatedVMNIC: + masterIfName := plugin.findInterfaceByMAC(opt.ifInfo.MacAddress.String()) + if masterIfName == "" { + err := plugin.Errorf("Failed to find the master interface") + return nil, err + } + logger.Info("Found master interface", zap.String("ifname", masterIfName)) + if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { + return nil, err + } + + nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) + if err != nil { + return nil, err + } + + nwInfo = network.NetworkInfo{ + Id: "azure-" + opt.ifInfo.MacAddress.String(), + Mode: opt.ipamAddConfig.nwCfg.Mode, + MasterIfName: masterIfName, + AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, + BridgeName: opt.ipamAddConfig.nwCfg.Bridge, + EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: see above + DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT: see above + NetNs: opt.ipamAddConfig.args.Netns, + Options: opt.ipamAddConfig.options, + DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, + ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: see above + NICType: string(cns.DelegatedVMNIC), + } + + if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { + logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) + return nil, err + } + setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) + + } + + // populate endpoint info // taken from createEndpointInternal function // populate endpoint info fields code is below // did not move cns client code because we don't need it here @@ -733,9 +817,8 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) return nil, err } - // TODO: (1/2) Possible solution to needing nwInfo for epInfo: move this code related to policies to after we populate the network info policyArgs := PolicyArgs{ - nwInfo: opt.nwInfo, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? getEndpointPolicies requires nwInfo.Subnets only (checked) + nwInfo: &nwInfo, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? getEndpointPolicies requires nwInfo.Subnets only (checked) nwCfg: opt.nwCfg, ipconfigs: defaultInterfaceInfo.IPConfigs, } @@ -752,8 +835,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. // IT will result in unpredictable behavior if API server decides to // reorder DELETE and ADD call for new incarnation of same POD. - // TODO: (2/2) Possible solution to needing nwInfo for epInfo: move this code related to setEndpointOptions to after we populate the network info - vethName = fmt.Sprintf("%s%s%s", opt.nwInfo.Id, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) The dummy nwInfo has the Id, right? + vethName = fmt.Sprintf("%s%s%s", nwInfo.Id, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) The dummy nwInfo has the Id, right? } epInfo = network.EndpointInfo{ @@ -762,7 +844,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn NetNsPath: opt.args.Netns, IfName: opt.args.IfName, Data: make(map[string]interface{}), - DNS: epDNSInfo, + EndpointDNS: epDNSInfo, Policies: opt.policies, IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, @@ -807,87 +889,6 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) } - // now need to populate the network info section of the ep info struct here (or you can do it in the switch statement above) - switch opt.ifInfo.NICType { - case cns.InfraNIC: - masterIfName := plugin.findMasterInterfaceBySubnet(opt.ipamAddConfig.nwCfg, &opt.ifInfo.HostSubnetPrefix) - if masterIfName == "" { - err := plugin.Errorf("Failed to find the master interface") - return nil, err - } - logger.Info("Found master interface", zap.String("ifname", masterIfName)) - if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { - return nil, err - } - - nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) - if err != nil { - return nil, err - } - - nwInfo = network.NetworkInfo{ - Id: opt.networkID, - Mode: opt.ipamAddConfig.nwCfg.Mode, - MasterIfName: masterIfName, - AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, - BridgeName: opt.ipamAddConfig.nwCfg.Bridge, - EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above - DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT TODO: Check what takes precedence - Policies: opt.policies, - NetNs: opt.ipamAddConfig.args.Netns, - Options: opt.ipamAddConfig.options, - DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // TODO: check if IPV6Mode field can be deprecated // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above - IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, - ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above - IsIPv6Enabled: opt.ipv6Enabled, - NICType: string(cns.InfraNIC), - } - - if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { - logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) - return nil, err - } - setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) - - case cns.DelegatedVMNIC: - masterIfName := plugin.findInterfaceByMAC(opt.ifInfo.MacAddress.String()) - if masterIfName == "" { - err := plugin.Errorf("Failed to find the master interface") - return nil, err - } - logger.Info("Found master interface", zap.String("ifname", masterIfName)) - if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { - return nil, err - } - - nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) - if err != nil { - return nil, err - } - - nwInfo = network.NetworkInfo{ - Id: "azure-" + opt.ifInfo.MacAddress.String(), - Mode: opt.ipamAddConfig.nwCfg.Mode, - MasterIfName: masterIfName, - AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, - BridgeName: opt.ipamAddConfig.nwCfg.Bridge, - EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: see above - DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT: see above - NetNs: opt.ipamAddConfig.args.Netns, - Options: opt.ipamAddConfig.options, - DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: see above - NICType: string(cns.DelegatedVMNIC), - } - - if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { - logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) - return nil, err - } - setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) - - } // populate endpoint info with network info fields // TODO: maybe instead of creating a network info, directly modify epInfo, but some helper methods rely on networkinfo... @@ -903,6 +904,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn epInfo.DisableHairpinOnHostInterface = nwInfo.DisableHairpinOnHostInterface epInfo.IPAMType = nwInfo.IPAMType epInfo.IsIPv6Enabled = nwInfo.IsIPv6Enabled + epInfo.NetworkDNS = nwInfo.DNS // now our ep info should have the full combined information from both the network and endpoint structs return &epInfo, nil @@ -924,7 +926,7 @@ func (plugin *NetPlugin) createEndpoint(epInfo *network.EndpointInfo, CNSUrl str Mode: epInfo.Mode, Subnets: epInfo.Subnets, PodSubnet: epInfo.PodSubnet, - DNS: epInfo.DNS, + DNS: epInfo.NetworkDNS, Policies: epInfo.Policies, BridgeName: epInfo.BridgeName, EnableSnatOnHost: epInfo.EnableSnatOnHost, @@ -1328,8 +1330,8 @@ func (plugin *NetPlugin) Get(args *cniSkel.CmdArgs) error { result.Routes = append(result.Routes, &cniTypes.Route{Dst: route.Dst, GW: route.Gw}) } - result.DNS.Nameservers = epInfo.DNS.Servers - result.DNS.Domain = epInfo.DNS.Suffix + result.DNS.Nameservers = epInfo.EndpointDNS.Servers + result.DNS.Domain = epInfo.EndpointDNS.Suffix return nil } diff --git a/network/endpoint.go b/network/endpoint.go index 9a89f42d2d..43c52597f3 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -66,7 +66,8 @@ type EndpointInfo struct { SandboxKey string IfIndex int MacAddress net.HardwareAddr - DNS DNSInfo + EndpointDNS DNSInfo + NetworkDNS DNSInfo IPAddresses []net.IPNet IPsToRouteViaHost []string InfraVnetIP net.IPNet @@ -278,7 +279,7 @@ func (ep *endpoint) getInfo() *EndpointInfo { MacAddress: ep.MacAddress, SandboxKey: ep.SandboxKey, IfIndex: 0, // Azure CNI supports only one interface - DNS: ep.DNS, + EndpointDNS: ep.DNS, EnableSnatOnHost: ep.EnableSnatOnHost, EnableInfraVnet: ep.EnableInfraVnet, EnableMultiTenancy: ep.EnableMultitenancy, diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index ce0e2e6e7e..509321a5d7 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -106,7 +106,7 @@ func (nw *network) newEndpointImpl( InfraVnetIP: defaultEpInfo.InfraVnetIP, LocalIP: localIP, IPAddresses: defaultEpInfo.IPAddresses, - DNS: defaultEpInfo.DNS, + DNS: defaultEpInfo.EndpointDNS, VlanID: vlanid, EnableSnatOnHost: defaultEpInfo.EnableSnatOnHost, EnableInfraVnet: defaultEpInfo.EnableInfraVnet, diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 823c7ad737..2108d0f05d 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -104,8 +104,8 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo, plc platform.ExecC hnsEndpoint := &hcsshim.HNSEndpoint{ Name: infraEpName, VirtualNetwork: nw.HnsId, - DNSSuffix: epInfo.DNS.Suffix, - DNSServerList: strings.Join(epInfo.DNS.Servers, ","), + DNSSuffix: epInfo.EndpointDNS.Suffix, + DNSServerList: strings.Join(epInfo.EndpointDNS.Servers, ","), Policies: policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy), } @@ -165,7 +165,7 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo, plc platform.ExecC IfName: epInfo.IfName, IPAddresses: epInfo.IPAddresses, Gateways: []net.IP{net.ParseIP(hnsResponse.GatewayAddress)}, - DNS: epInfo.DNS, + DNS: epInfo.EndpointDNS, VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, NetNs: epInfo.NetNsPath, @@ -213,9 +213,9 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE Name: infraEpName, HostComputeNetwork: nw.HnsId, Dns: hcn.Dns{ - Search: strings.Split(epInfo.DNS.Suffix, ","), - ServerList: epInfo.DNS.Servers, - Options: epInfo.DNS.Options, + Search: strings.Split(epInfo.EndpointDNS.Suffix, ","), + ServerList: epInfo.EndpointDNS.Servers, + Options: epInfo.EndpointDNS.Options, }, SchemaVersion: hcn.SchemaVersion{ Major: hcnSchemaVersionMajor, @@ -390,7 +390,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( IfName: epInfo.IfName, IPAddresses: epInfo.IPAddresses, Gateways: []net.IP{gateway}, - DNS: epInfo.DNS, + DNS: epInfo.EndpointDNS, VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, NetNs: epInfo.NetNsPath, diff --git a/network/endpoint_windows_test.go b/network/endpoint_windows_test.go index 4e65c6a2ba..a352b7316f 100644 --- a/network/endpoint_windows_test.go +++ b/network/endpoint_windows_test.go @@ -32,7 +32,7 @@ func TestNewAndDeleteEndpointImplHnsV2(t *testing.T) { NetNsPath: "fakeNameSpace", IfName: "eth0", Data: make(map[string]interface{}), - DNS: DNSInfo{ + EndpointDNS: DNSInfo{ Suffix: "10.0.0.0", Servers: []string{"10.0.0.1, 10.0.0.2"}, Options: nil, @@ -76,7 +76,7 @@ func TestNewEndpointImplHnsv2Timesout(t *testing.T) { NetNsPath: "fakeNameSpace", IfName: "eth0", Data: make(map[string]interface{}), - DNS: DNSInfo{ + EndpointDNS: DNSInfo{ Suffix: "10.0.0.0", Servers: []string{"10.0.0.1, 10.0.0.2"}, Options: nil, @@ -103,7 +103,7 @@ func TestDeleteEndpointImplHnsv2Timeout(t *testing.T) { NetNsPath: "fakeNameSpace", IfName: "eth0", Data: make(map[string]interface{}), - DNS: DNSInfo{ + EndpointDNS: DNSInfo{ Suffix: "10.0.0.0", Servers: []string{"10.0.0.1, 10.0.0.2"}, Options: nil, @@ -152,7 +152,7 @@ func TestCreateEndpointImplHnsv1Timeout(t *testing.T) { NetNsPath: "fakeNameSpace", IfName: "eth0", Data: make(map[string]interface{}), - DNS: DNSInfo{ + EndpointDNS: DNSInfo{ Suffix: "10.0.0.0", Servers: []string{"10.0.0.1, 10.0.0.2"}, Options: nil, @@ -179,7 +179,7 @@ func TestDeleteEndpointImplHnsv1Timeout(t *testing.T) { NetNsPath: "fakeNameSpace", IfName: "eth0", Data: make(map[string]interface{}), - DNS: DNSInfo{ + EndpointDNS: DNSInfo{ Suffix: "10.0.0.0", Servers: []string{"10.0.0.1, 10.0.0.2"}, Options: nil, diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index 3e10db761c..12bd40ab80 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -28,7 +28,7 @@ func (client *OVSEndpointClient) NewSnatClient(snatBridgeIP, localIP string, epI localIP, snatBridgeIP, client.hostPrimaryMac, - epInfo.DNS.Servers, + epInfo.EndpointDNS.Servers, false, client.netlink, client.plClient, diff --git a/network/transparent_vlan_endpoint_snatroute_linux.go b/network/transparent_vlan_endpoint_snatroute_linux.go index b6f712758a..d997ead960 100644 --- a/network/transparent_vlan_endpoint_snatroute_linux.go +++ b/network/transparent_vlan_endpoint_snatroute_linux.go @@ -16,7 +16,7 @@ func (client *TransparentVlanEndpointClient) NewSnatClient(snatBridgeIP, localIP localIP, snatBridgeIP, client.hostPrimaryMac.String(), - epInfo.DNS.Servers, + epInfo.EndpointDNS.Servers, true, client.netlink, client.plClient, From 7e9282369a7917a4744e840815d4f208e3053f3d Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 28 Mar 2024 21:10:45 -0700 Subject: [PATCH 017/102] fix testIpamAddFail ut referencing wrong redeclared err variable, fix error message --- cni/network/network.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 0334887935..81531e81ac 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -597,13 +597,14 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { ipamAddConfig: &ipamAddConfig, ipv6Enabled: ipamAddResult.ipv6Enabled, } - epInfo, err := plugin.createEpInfo(&createEpInfoOpt) + var epInfo *network.EndpointInfo + epInfo, err = plugin.createEpInfo(&createEpInfoOpt) if err != nil { - return errors.Wrap(err, "failed to populate endpoint info struct") + return err } err = plugin.createEndpoint(epInfo, nwCfg.CNSUrl) if err != nil { - return errors.Wrap(err, "failed to create endpoint") + return err } // TODO: should this statement be based on the current iteration instead of the constant ifIndex? sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", From 8abd9043d3699f13e3cf3efe8fb26ef8ca816833 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 29 Mar 2024 13:20:44 -0400 Subject: [PATCH 018/102] UT fix part one --- cni/network/invoker_azure.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index 4c3ca88103..b891ab9dd1 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -70,15 +70,6 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er return addResult, err } - addResult.interfaceInfo = []network.InterfaceInfo{} - addResult.interfaceInfo = append(addResult.interfaceInfo, network.InterfaceInfo{ - HostSubnetPrefix: net.IPNet{}, - }) - - if len(result.IPs) > 0 { - addResult.interfaceInfo[0].HostSubnetPrefix = result.IPs[0].Address - } - defer func() { if err != nil { if len(addResult.interfaceInfo[0].IPConfigs) > 0 { @@ -131,6 +122,11 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er }, NICType: cns.InfraNIC, }) + + if len(result.IPs) > 0 { + addResult.interfaceInfo[0].HostSubnetPrefix = result.IPs[0].Address + } + return addResult, err } From ddffc9deb2c9897e5c27fb6c531b536b2aab9a58 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Mon, 1 Apr 2024 17:24:55 -0400 Subject: [PATCH 019/102] fix the getNetworkID and getNetworkInfo --- cni/network/invoker.go | 1 - cni/network/network.go | 72 +++++++++++++++++++--------------- cni/network/network_linux.go | 2 +- cni/network/network_windows.go | 22 ++++++++--- network/endpoint.go | 2 +- 5 files changed, 59 insertions(+), 40 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 4decc1c1e3..ea76889213 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -26,7 +26,6 @@ type IPAMAddConfig struct { } type IPAMAddResult struct { - // Splitting defaultInterfaceInfo from secondaryInterfacesInfo so we don't need to loop for default CNI result every time interfaceInfo []network.InterfaceInfo ipv6Enabled bool } diff --git a/cni/network/network.go b/cni/network/network.go index 81531e81ac..2e24169c53 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -335,11 +335,20 @@ func (plugin *NetPlugin) addIpamInvoker(ipamAddConfig IPAMAddConfig) (IPAMAddRes return ipamAddResult, nil } +// get network +func (plugin *NetPlugin) getNetworkID(netNs string, interfaceInfo *network.InterfaceInfo, nwCfg *cni.NetworkConfig) (string, error) { + networkID, err := plugin.getNetworkName(netNs, interfaceInfo, nwCfg) + if err != nil { + return "", err + } + return networkID, nil +} + // get network info -func (plugin *NetPlugin) getNetworkInfo(netNs string, ipamAddResult IPAMAddResult, nwCfg *cni.NetworkConfig) (network.NetworkInfo, string, error) { - networkID, _ := plugin.getNetworkName(netNs, &ipamAddResult, nwCfg) - nwInfo, nwError := plugin.nm.GetNetworkInfo(networkID) - return nwInfo, networkID, nwError +func (plugin *NetPlugin) getNetworkInfo(netNs string, interfaceInfo *network.InterfaceInfo, nwCfg *cni.NetworkConfig) network.NetworkInfo { + networkID, _ := plugin.getNetworkID(netNs, interfaceInfo, nwCfg) + nwInfo, _ := plugin.nm.GetNetworkInfo(networkID) + return nwInfo } // CNI implementation @@ -472,7 +481,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { var ( nwInfo network.NetworkInfo networkID string - nwInfoErr error ) if nwCfg.MultiTenancy { @@ -507,7 +515,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { case network.AzureCNS: plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) default: - nwInfo, networkID, nwInfoErr = plugin.getNetworkInfo(args.Netns, ipamAddResult, nwCfg) + nwInfo = plugin.getNetworkInfo(args.Netns, nil, nwCfg) plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) } } @@ -523,45 +531,47 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } // This proably needs to be changed as we return all interfaces... sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) - } ifIndex, _ := findDefaultInterface(ipamAddResult) endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) + // if !nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS { + // nwInfo, nwInfoErr = plugin.getNetworkInfo(args.Netns, ipamAddResult, nwCfg) + // } + // Check whether the network already exists. - nwInfo, networkID, nwInfoErr = plugin.getNetworkInfo(args.Netns, ipamAddResult, nwCfg) // Handle consecutive ADD calls for infrastructure containers. // This is a temporary work around for issue #57253 of Kubernetes. // We can delete this if statement once they fix it. // Issue link: https://github.com/kubernetes/kubernetes/issues/57253 - if nwInfoErr == nil { - logger.Info("Found network with subnet", - zap.String("network", networkID), - zap.String("subnet", nwInfo.Subnets[0].Prefix.String())) - nwInfo.IPAMType = nwCfg.IPAM.Type - options = nwInfo.Options - - var resultSecondAdd *cniTypesCurr.Result - resultSecondAdd, err = plugin.handleConsecutiveAdd(args, endpointID, networkID, &nwInfo, nwCfg) - if err != nil { - logger.Error("handleConsecutiveAdd failed", zap.Error(err)) - return err - } + // if nwInfoErr == nil { + // logger.Info("Found network with subnet", + // zap.String("network", networkID), + // zap.String("subnet", nwInfo.Subnets[0].Prefix.String())) + // nwInfo.IPAMType = nwCfg.IPAM.Type + // options = nwInfo.Options + + // var resultSecondAdd *cniTypesCurr.Result + // resultSecondAdd, err = plugin.handleConsecutiveAdd(args, endpointID, networkID, &nwInfo, nwCfg) + // if err != nil { + // logger.Error("handleConsecutiveAdd failed", zap.Error(err)) + // return err + // } - if resultSecondAdd != nil { - ifIndex, err = findDefaultInterface(ipamAddResult) - if err != nil { - ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{}) - ifIndex = len(ipamAddResult.interfaceInfo) - 1 - } - ipamAddResult.interfaceInfo[ifIndex] = convertCniResultToInterfaceInfo(resultSecondAdd) - return nil - } - } + // if resultSecondAdd != nil { + // ifIndex, err = findDefaultInterface(ipamAddResult) + // if err != nil { + // ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{}) + // ifIndex = len(ipamAddResult.interfaceInfo) - 1 + // } + // ipamAddResult.interfaceInfo[ifIndex] = convertCniResultToInterfaceInfo(resultSecondAdd) + // return nil + // } + // } defer func() { //nolint:gocritic if err != nil { diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 19c134a73b..fd0a8417b1 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -119,7 +119,7 @@ func addIPV6EndpointPolicy(nwInfo network.NetworkInfo) (policy.Policy, error) { return policy.Policy{}, nil } -func (plugin *NetPlugin) getNetworkName(_ string, _ *IPAMAddResult, nwCfg *cni.NetworkConfig) (string, error) { +func (plugin *NetPlugin) getNetworkName(_ string, _ *network.InterfaceInfo, nwCfg *cni.NetworkConfig) (string, error) { return nwCfg.Name, nil } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 1fb2e4936c..bf325e438f 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -142,8 +142,19 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { } -func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResult, nwCfg *cni.NetworkConfig) (string, error) { +func (plugin *NetPlugin) getNetworkName(netNs string, interfaceInfo *network.InterfaceInfo, nwCfg *cni.NetworkConfig) (string, error) { + var err error + // Swiftv2 path => interfaceInfo.NICType = delegated NIC + // For singletenancy => nwCfg.Name + // Swiftv1 => interfaceInfo.NCResponse != nil && ipamAddResult != nil + determineWinVer() + // Swiftv2 L1VH Network Name + if interfaceInfo != nil && interfaceInfo.NICType == cns.DelegatedVMNIC { + logger.Info("swiftv2", zap.String("network name", interfaceInfo.MacAddress.String())) + return "azure-" + interfaceInfo.MacAddress.String(), nil + } + // For singletenancy, the network name is simply the nwCfg.Name if !nwCfg.MultiTenancy { return nwCfg.Name, nil @@ -156,16 +167,15 @@ func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResu // First try to build the network name from the cnsResponse if present // This will happen during ADD call - ifIndex, err := findDefaultInterface(*ipamAddResult) - if ipamAddResult != nil && ipamAddResult.interfaceInfo[ifIndex].NCResponse != nil { - // find defaultInterface within AddResult + // ifIndex, err := findDefaultInterface(*ipamAddResult) + if interfaceInfo != nil && interfaceInfo.NCResponse != nil { // swiftv1 path if err != nil { logger.Error("Error finding InfraNIC interface", zap.Error(err)) return "", errors.Wrap(err, "cns did not return an InfraNIC") } // networkName will look like ~ azure-vlan1-172-28-1-0_24 - ipAddrNet := ipamAddResult.interfaceInfo[ifIndex].IPConfigs[0].Address + ipAddrNet := interfaceInfo.IPConfigs[0].Address prefix, err := netip.ParsePrefix(ipAddrNet.String()) if err != nil { logger.Error("Error parsing network CIDR", @@ -175,7 +185,7 @@ func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResu } networkName := strings.ReplaceAll(prefix.Masked().String(), ".", "-") networkName = strings.ReplaceAll(networkName, "/", "_") - networkName = fmt.Sprintf("%s-vlan%v-%v", nwCfg.Name, ipamAddResult.interfaceInfo[ifIndex].NCResponse.MultiTenancyInfo.ID, networkName) + networkName = fmt.Sprintf("%s-vlan%v-%v", nwCfg.Name, interfaceInfo.NCResponse.MultiTenancyInfo.ID, networkName) return networkName, nil } diff --git a/network/endpoint.go b/network/endpoint.go index 43c52597f3..74f8785888 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -137,7 +137,7 @@ type InterfaceInfo struct { NICType cns.NICType SkipDefaultRoutes bool HostSubnetPrefix net.IPNet // Move this field from ipamAddResult - NCResponse *cns.GetNetworkContainerResponse // TODO: create a new struct and keep fields that are being used + NCResponse *cns.GetNetworkContainerResponse // TODO: create a new struct and keep fields that are being used } type IPConfig struct { From e5ed570285412670dc3dc29acc28ca14501c32f6 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 1 Apr 2024 15:32:37 -0700 Subject: [PATCH 020/102] move create endpoint to network package, remove ifIndex as needed --- cni/network/network.go | 73 +++--------- network/endpoint.go | 10 +- network/endpoint_linux.go | 215 ++++++++++++++++++------------------ network/endpoint_windows.go | 9 +- network/manager.go | 61 ++++++---- network/manager_mock.go | 23 ++-- network/network.go | 84 ++++++++++++++ 7 files changed, 267 insertions(+), 208 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 2e24169c53..132d8d1368 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -524,6 +524,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } + //TODO: remove ifIndex, _ := findDefaultInterface(ipamAddResult) if ifIndex == -1 { @@ -584,6 +585,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // TODO: is it possible nwInfo is not populated? YES it seems so, in which case the nwInfo is an empty struct! // TODO: This will mean when we try to do nwInfo.Id or nwInfo.Subnets in createEpInfo, we'll have problems! + epInfos := []*network.EndpointInfo{} for _, ifInfo := range ipamAddResult.interfaceInfo { // TODO: hopefully I can get natInfo here? natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) @@ -612,15 +614,26 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return err } - err = plugin.createEndpoint(epInfo, nwCfg.CNSUrl) + //ep, err = plugin.createEndpoint(epInfo, nwCfg.CNSUrl) if err != nil { return err } + // TODO: is this how we choose which ep to use? + // the endpoint that is saved in the statefile is represented fully by the ep info/ep with the InfraNIC type + if ifInfo.NICType == cns.InfraNIC { + //saveEp = ep + } + epInfos = append(epInfos, epInfo) // TODO: should this statement be based on the current iteration instead of the constant ifIndex? sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) } + cnsclient, err := cnscli.New(nwCfg.CNSUrl, defaultRequestTimeout) + if err != nil { + return errors.Wrap(err, "failed to create cns client") + } + plugin.nm.EndpointCreate(cnsclient, epInfos) // // Create network // if nwInfoErr != nil { @@ -829,6 +842,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn return nil, err } policyArgs := PolicyArgs{ + // pass podsubnet info etc. part of epinfo nwInfo: &nwInfo, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? getEndpointPolicies requires nwInfo.Subnets only (checked) nwCfg: opt.nwCfg, ipconfigs: defaultInterfaceInfo.IPConfigs, @@ -921,63 +935,6 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn return &epInfo, nil } -// networkID is opt.nwInfo.ID -// cns url is opt.nwCfg.CNSUrl -// actually creates the network and corresponding endpoint -func (plugin *NetPlugin) createEndpoint(epInfo *network.EndpointInfo, CNSUrl string) error { - // check if network exists - nwInfo, nwGetErr := plugin.nm.GetNetworkInfo(epInfo.NetworkId) - if nwGetErr != nil { - // Create the network if it is not found - // populate network info with fields from ep info - nwInfo = network.NetworkInfo{ - MasterIfName: epInfo.MasterIfName, - AdapterName: epInfo.AdapterName, - Id: epInfo.NetworkId, - Mode: epInfo.Mode, - Subnets: epInfo.Subnets, - PodSubnet: epInfo.PodSubnet, - DNS: epInfo.NetworkDNS, - Policies: epInfo.Policies, - BridgeName: epInfo.BridgeName, - EnableSnatOnHost: epInfo.EnableSnatOnHost, - NetNs: epInfo.NetNs, - Options: epInfo.Options, - DisableHairpinOnHostInterface: epInfo.DisableHairpinOnHostInterface, - IPV6Mode: epInfo.IPV6Mode, - IPAMType: epInfo.IPAMType, - ServiceCidrs: epInfo.ServiceCidrs, - IsIPv6Enabled: epInfo.IsIPv6Enabled, - NICType: string(epInfo.NICType), - } - err := plugin.nm.CreateNetwork(&nwInfo) - if err != nil { - err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) - return err // added - } - } - - cnsclient, err := cnscli.New(CNSUrl, defaultRequestTimeout) - if err != nil { - logger.Error("failed to initialized cns client", zap.String("url", CNSUrl), - zap.String("error", err.Error())) - return plugin.Errorf(err.Error()) - } - - // Create the endpoint. - logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) - sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) - // TODO: pass in network id, or get network id from ep info-- it makes more sense if it came from epInfo - // since the network is directly related to the epInfo - // TODO: okay to pass in a slice of epInfo this way and just say epIndex is 0, or is refactoring create endpoint needed? - err = plugin.nm.CreateEndpoint(cnsclient, nwInfo.Id, []*network.EndpointInfo{epInfo}, 0) - if err != nil { - err = plugin.Errorf("Failed to create endpoint: %v", err) - return err //added - } - return nil -} - // cleanup allocated ipv4 and ipv6 addresses if they exist func (plugin *NetPlugin) cleanupAllocationOnError( result []*network.IPConfig, diff --git a/network/endpoint.go b/network/endpoint.go index 74f8785888..04dceae050 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -113,6 +113,8 @@ type EndpointInfo struct { // ServiceCidrs omitted IsIPv6Enabled bool // NICType omitted + + IfInfo *InterfaceInfo // for secondary interfaces when creating endpoint struct } // RouteInfo contains information about an IP route. @@ -164,26 +166,24 @@ func (nw *network) newEndpoint( netioCli netio.NetIOInterface, nsc NamespaceClientInterface, iptc ipTablesClient, - epInfo []*EndpointInfo, - epIndex int, + epInfo *EndpointInfo, ) (*endpoint, error) { var ep *endpoint var err error defer func() { if err != nil { - logger.Error("Failed to create endpoint with err", zap.String("id", epInfo[epIndex].Id), zap.Error(err)) + logger.Error("Failed to create endpoint with err", zap.String("id", epInfo.Id), zap.Error(err)) } }() // Call the platform implementation. // Pass nil for epClient and will be initialized in newendpointImpl - ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, nsc, iptc, epInfo, epIndex) + ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, nsc, iptc, epInfo) if err != nil { return nil, err } - nw.Endpoints[ep.Id] = ep logger.Info("Created endpoint. Num of endpoints", zap.Any("ep", ep), zap.Int("numEndpoints", len(nw.Endpoints))) return ep, nil } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 509321a5d7..ffe9742f23 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -57,8 +57,7 @@ func (nw *network) newEndpointImpl( testEpClient EndpointClient, nsc NamespaceClientInterface, iptc ipTablesClient, - epInfo []*EndpointInfo, - epIndex int, + epInfo *EndpointInfo, ) (*endpoint, error) { var ( err error @@ -66,7 +65,7 @@ func (nw *network) newEndpointImpl( contIfName string localIP string vlanid = 0 - defaultEpInfo = epInfo[epIndex] + defaultEpInfo = epInfo containerIf *net.Interface ) @@ -124,134 +123,132 @@ func (nw *network) newEndpointImpl( ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} } - for _, epInfo := range epInfo { - // testEpClient is non-nil only when the endpoint is created for the unit test - // resetting epClient to testEpClient in loop to use the test endpoint client if specified - epClient := testEpClient - if epClient == nil { - //nolint:gocritic - if vlanid != 0 { - if nw.Mode == opModeTransparentVlan { - logger.Info("Transparent vlan client") - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) - } - epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc, nsc, iptc) - } else { - logger.Info("OVS client") - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) - } - - epClient = NewOVSEndpointClient( - nw, - epInfo, - hostIfName, - contIfName, - vlanid, - localIP, - nl, - ovsctl.NewOvsctl(), - plc, - iptc) + // testEpClient is non-nil only when the endpoint is created for the unit test + // resetting epClient to testEpClient in loop to use the test endpoint client if specified + epClient := testEpClient + if epClient == nil { + //nolint:gocritic + if vlanid != 0 { + if nw.Mode == opModeTransparentVlan { + logger.Info("Transparent vlan client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) } - } else if nw.Mode != opModeTransparent { - logger.Info("Bridge client") - epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) - } else if epInfo.NICType == cns.DelegatedVMNIC { - logger.Info("Secondary client") - epClient = NewSecondaryEndpointClient(nl, netioCli, plc, nsc, ep) + epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc, nsc, iptc) } else { - logger.Info("Transparent client") - epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, netioCli, plc) + logger.Info("OVS client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + } + + epClient = NewOVSEndpointClient( + nw, + epInfo, + hostIfName, + contIfName, + vlanid, + localIP, + nl, + ovsctl.NewOvsctl(), + plc, + iptc) } + } else if nw.Mode != opModeTransparent { + logger.Info("Bridge client") + epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) + } else if epInfo.NICType == cns.DelegatedVMNIC { + logger.Info("Secondary client") + epClient = NewSecondaryEndpointClient(nl, netioCli, plc, nsc, ep) + } else { + logger.Info("Transparent client") + epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, netioCli, plc) } + } - //nolint:gocritic - defer func(client EndpointClient, contIfName string) { - // Cleanup on failure. - if err != nil { - logger.Error("CNI error. Delete Endpoint and rules that are created", zap.Error(err), zap.String("contIfName", contIfName)) - if containerIf != nil { - client.DeleteEndpointRules(ep) - } - // set deleteHostVeth to true to cleanup host veth interface if created - //nolint:errcheck // ignore error - client.DeleteEndpoints(ep) + //nolint:gocritic + defer func(client EndpointClient, contIfName string) { + // Cleanup on failure. + if err != nil { + logger.Error("CNI error. Delete Endpoint and rules that are created", zap.Error(err), zap.String("contIfName", contIfName)) + if containerIf != nil { + client.DeleteEndpointRules(ep) } - }(epClient, contIfName) + // set deleteHostVeth to true to cleanup host veth interface if created + //nolint:errcheck // ignore error + client.DeleteEndpoints(ep) + } + }(epClient, contIfName) + + // wrapping endpoint client commands in anonymous func so that namespace can be exit and closed before the next loop + //nolint:wrapcheck // ignore wrap check + err = func() error { + if epErr := epClient.AddEndpoints(epInfo); epErr != nil { + return epErr + } - // wrapping endpoint client commands in anonymous func so that namespace can be exit and closed before the next loop - //nolint:wrapcheck // ignore wrap check - err = func() error { - if epErr := epClient.AddEndpoints(epInfo); epErr != nil { + if epInfo.NICType == cns.InfraNIC { + var epErr error + containerIf, epErr = netioCli.GetNetworkInterfaceByName(contIfName) + if epErr != nil { return epErr } + ep.MacAddress = containerIf.HardwareAddr + } - if epInfo.NICType == cns.InfraNIC { - var epErr error - containerIf, epErr = netioCli.GetNetworkInterfaceByName(contIfName) - if epErr != nil { - return epErr - } - ep.MacAddress = containerIf.HardwareAddr - } + // Setup rules for IP addresses on the container interface. + if epErr := epClient.AddEndpointRules(epInfo); epErr != nil { + return epErr + } - // Setup rules for IP addresses on the container interface. - if epErr := epClient.AddEndpointRules(epInfo); epErr != nil { + // If a network namespace for the container interface is specified... + if epInfo.NetNsPath != "" { + // Open the network namespace. + logger.Info("Opening netns", zap.Any("NetNsPath", epInfo.NetNsPath)) + ns, epErr := nsc.OpenNamespace(epInfo.NetNsPath) + if epErr != nil { return epErr } + defer ns.Close() - // If a network namespace for the container interface is specified... - if epInfo.NetNsPath != "" { - // Open the network namespace. - logger.Info("Opening netns", zap.Any("NetNsPath", epInfo.NetNsPath)) - ns, epErr := nsc.OpenNamespace(epInfo.NetNsPath) - if epErr != nil { - return epErr - } - defer ns.Close() - - if epErr := epClient.MoveEndpointsToContainerNS(epInfo, ns.GetFd()); epErr != nil { - return epErr - } - - // Enter the container network namespace. - logger.Info("Entering netns", zap.Any("NetNsPath", epInfo.NetNsPath)) - if epErr := ns.Enter(); epErr != nil { - return epErr - } - - // Return to host network namespace. - defer func() { - logger.Info("Exiting netns", zap.Any("NetNsPath", epInfo.NetNsPath)) - if epErr := ns.Exit(); epErr != nil { - logger.Error("Failed to exit netns with", zap.Error(epErr)) - } - }() + if epErr := epClient.MoveEndpointsToContainerNS(epInfo, ns.GetFd()); epErr != nil { + return epErr } - if epInfo.IPV6Mode != "" { - // Enable ipv6 setting in container - logger.Info("Enable ipv6 setting in container.") - nuc := networkutils.NewNetworkUtils(nl, plc) - if epErr := nuc.UpdateIPV6Setting(0); epErr != nil { - return fmt.Errorf("Enable ipv6 in container failed:%w", epErr) - } + // Enter the container network namespace. + logger.Info("Entering netns", zap.Any("NetNsPath", epInfo.NetNsPath)) + if epErr := ns.Enter(); epErr != nil { + return epErr } - // If a name for the container interface is specified... - if epInfo.IfName != "" { - if epErr := epClient.SetupContainerInterfaces(epInfo); epErr != nil { - return epErr + // Return to host network namespace. + defer func() { + logger.Info("Exiting netns", zap.Any("NetNsPath", epInfo.NetNsPath)) + if epErr := ns.Exit(); epErr != nil { + logger.Error("Failed to exit netns with", zap.Error(epErr)) } + }() + } + + if epInfo.IPV6Mode != "" { + // Enable ipv6 setting in container + logger.Info("Enable ipv6 setting in container.") + nuc := networkutils.NewNetworkUtils(nl, plc) + if epErr := nuc.UpdateIPV6Setting(0); epErr != nil { + return fmt.Errorf("Enable ipv6 in container failed:%w", epErr) } + } - return epClient.ConfigureContainerInterfacesAndRoutes(epInfo) - }() - if err != nil { - return nil, err + // If a name for the container interface is specified... + if epInfo.IfName != "" { + if epErr := epClient.SetupContainerInterfaces(epInfo); epErr != nil { + return epErr + } } + + return epClient.ConfigureContainerInterfacesAndRoutes(epInfo) + }() + if err != nil { + return nil, err } return ep, nil diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 2108d0f05d..200682e994 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -73,19 +73,18 @@ func (nw *network) newEndpointImpl( _ EndpointClient, _ NamespaceClientInterface, _ ipTablesClient, - epInfo []*EndpointInfo, - epIndex int, + epInfo *EndpointInfo, ) (*endpoint, error) { // there is only 1 epInfo for windows, multiple interfaces will be added in the future - if useHnsV2, err := UseHnsV2(epInfo[epIndex].NetNsPath); useHnsV2 { + if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { if err != nil { return nil, err } - return nw.newEndpointImplHnsV2(cli, epInfo[epIndex]) + return nw.newEndpointImplHnsV2(cli, epInfo) } - return nw.newEndpointImplHnsV1(epInfo[epIndex], plc) + return nw.newEndpointImplHnsV1(epInfo, plc) } // newEndpointImplHnsV1 creates a new endpoint in the network using HnsV1 diff --git a/network/manager.go b/network/manager.go index 303e812ea9..3527e41511 100644 --- a/network/manager.go +++ b/network/manager.go @@ -98,7 +98,8 @@ type NetworkManager interface { FindNetworkIDFromNetNs(netNs string) (string, error) GetNumEndpointsByContainerID(containerID string) int - CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo, epIndex int) error + CreateEndpoint(client apipaClient, networkID string, epInfo *EndpointInfo) (*endpoint, error) + EndpointCreate(client apipaClient, epInfos []*EndpointInfo) error // TODO: change name DeleteEndpoint(networkID string, endpointID string, epInfo *EndpointInfo) error GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) @@ -109,6 +110,7 @@ type NetworkManager interface { GetNumberOfEndpoints(ifName string, networkID string) int GetEndpointID(containerID, ifName string) string IsStatelessCNIMode() bool + SaveState(networkID string, ep *endpoint) error } // Creates a new network manager. @@ -299,11 +301,6 @@ func (nm *networkManager) AddExternalInterface(ifName string, subnet string) err return err } - err = nm.save() - if err != nil { - return err - } - return nil } @@ -317,11 +314,6 @@ func (nm *networkManager) CreateNetwork(nwInfo *NetworkInfo) error { return err } - err = nm.save() - if err != nil { - return err - } - return nil } @@ -372,25 +364,25 @@ func (nm *networkManager) GetNetworkInfo(networkId string) (NetworkInfo, error) } // CreateEndpoint creates a new container endpoint. -func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo []*EndpointInfo, epIndex int) error { +func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo *EndpointInfo) (*endpoint, error) { nm.Lock() defer nm.Unlock() nw, err := nm.getNetwork(networkID) if err != nil { - return err + return nil, err } if nw.VlanId != 0 { - if epInfo[epIndex].Data[VlanIDKey] == nil { + if epInfo.Data[VlanIDKey] == nil { logger.Info("overriding endpoint vlanid with network vlanid") - epInfo[epIndex].Data[VlanIDKey] = nw.VlanId + epInfo.Data[VlanIDKey] = nw.VlanId } } - ep, err := nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, nm.iptablesClient, epInfo, epIndex) + ep, err := nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, nm.iptablesClient, epInfo) if err != nil { - return err + return nil, err } // any error after this point should also clean up the endpoint we created above defer func() { @@ -406,15 +398,13 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn if nm.IsStatelessCNIMode() { err = nm.UpdateEndpointState(ep) - return err - } - - err = nm.save() - if err != nil { - return err + if err != nil { + return nil, err + } + // TODO: If stateless cni success, we still return ep right } - return nil + return ep, nil } // UpdateEndpointState will make a call to CNS updatEndpointState API in the stateless CNI mode @@ -697,3 +687,26 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string { } return containerID + "-" + ifName } +func (nm *networkManager) SaveState(networkID string, ep *endpoint) error { + // TODO: Necessary? + nm.Lock() + defer nm.Unlock() + + nw, err := nm.getNetwork(networkID) + if err != nil { + return err + } + + nw.Endpoints[ep.Id] = ep // used only once + + if nm.IsStatelessCNIMode() { + err = nm.UpdateEndpointState(ep) + return err + } + + err = nm.save() + if err != nil { + return err + } + return nil +} diff --git a/network/manager_mock.go b/network/manager_mock.go index 4d4675debe..14b5df3281 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -53,15 +53,15 @@ func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (NetworkInfo, err } // CreateEndpoint mock -func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfos []*EndpointInfo, _ int) error { - for _, epInfo := range epInfos { - if err := nm.TestEndpointClient.AddEndpoints(epInfo); err != nil { - return err - } +// TODO: Fix mock behavior because create endpoint no longer also saves the state +func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfo *EndpointInfo) (*endpoint, error) { + + if err := nm.TestEndpointClient.AddEndpoints(epInfo); err != nil { + return nil, err } - nm.TestEndpointInfoMap[epInfos[0].Id] = epInfos[0] - return nil + nm.TestEndpointInfoMap[epInfo.Id] = epInfo + return &endpoint{}, nil } // DeleteEndpoint mock @@ -154,3 +154,12 @@ func (nm *MockNetworkManager) GetNumEndpointsByContainerID(_ string) int { return numEndpoints } + +// TODO: understand mock behavior +func (nm *MockNetworkManager) SaveState(networkID string, ep *endpoint) error { + return nil +} + +func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfo []*EndpointInfo) error { + return nil +} diff --git a/network/network.go b/network/network.go index 7caab89fce..74b4e29182 100644 --- a/network/network.go +++ b/network/network.go @@ -8,6 +8,7 @@ import ( "net" "strings" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" "go.uber.org/zap" @@ -297,3 +298,86 @@ func (nm *networkManager) GetNumEndpointsByContainerID(containerID string) int { return numEndpoints } + +// networkID is opt.nwInfo.ID +// cns url is opt.nwCfg.CNSUrl +// actually creates the network and corresponding endpoint +func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*EndpointInfo) error { + interfaceInfos := []*InterfaceInfo{} + var networkIDofChosenEndpoint string + var epToSave *endpoint + for _, epInfo := range epInfos { + // check if network exists + nwInfo, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkId) + if nwGetErr != nil { + // Create the network if it is not found + // populate network info with fields from ep info + nwInfo = NetworkInfo{ + MasterIfName: epInfo.MasterIfName, + AdapterName: epInfo.AdapterName, + Id: epInfo.NetworkId, + Mode: epInfo.Mode, + Subnets: epInfo.Subnets, + PodSubnet: epInfo.PodSubnet, + DNS: epInfo.NetworkDNS, + Policies: epInfo.Policies, + BridgeName: epInfo.BridgeName, + EnableSnatOnHost: epInfo.EnableSnatOnHost, + NetNs: epInfo.NetNs, + Options: epInfo.Options, + DisableHairpinOnHostInterface: epInfo.DisableHairpinOnHostInterface, + IPV6Mode: epInfo.IPV6Mode, + IPAMType: epInfo.IPAMType, + ServiceCidrs: epInfo.ServiceCidrs, + IsIPv6Enabled: epInfo.IsIPv6Enabled, + NICType: string(epInfo.NICType), + } + err := nm.CreateNetwork(&nwInfo) + if err != nil { + // TODO: error messages/handling are different in this file + // err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) + return err // added + } + } + + // Create the endpoint. + logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) + // sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) + // TODO: pass in network id, or get network id from ep info-- it makes more sense if it came from epInfo + // since the network is directly related to the epInfo + // TODO: okay to pass in a slice of epInfo this way and just say epIndex is 0, or is refactoring create endpoint needed? + ep, err := nm.CreateEndpoint(cnsclient, nwInfo.Id, epInfo) + if err != nil { + // err = plugin.Errorf("Failed to create endpoint: %v", err) + return err //added + } + // if infra nic, we will use this endpoint for its root fields, and otherwise, we collect all secondary/delegated if infos to add to the ep secondary interface slice + if epInfo.NICType == cns.InfraNIC { + epToSave = ep + networkIDofChosenEndpoint = epInfo.NetworkId + } else { + // taken from secondary endpoint client + ipconfigs := make([]*IPConfig, len(epInfo.IPAddresses)) + for i, ipconfig := range epInfo.IPAddresses { + ipconfigs[i] = &IPConfig{Address: ipconfig} + } + + interfaceInfos = append(interfaceInfos, &InterfaceInfo{ + Name: epInfo.IfName, + MacAddress: epInfo.MacAddress, + IPConfigs: ipconfigs, + NICType: epInfo.NICType, + SkipDefaultRoutes: epInfo.SkipDefaultRoutes, + }) + } + } + // convert to one endpoint + epToSave.SecondaryInterfaces = map[string]*InterfaceInfo{} + for _, ifInfo := range interfaceInfos { + epToSave.SecondaryInterfaces[ifInfo.Name] = ifInfo + } + + // save + nm.SaveState(networkIDofChosenEndpoint, epToSave) + return nil +} From bb2e9799c51a55deb9f2371d42f345f54b0e539a Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 1 Apr 2024 16:38:52 -0700 Subject: [PATCH 021/102] use function to get network id --- cni/network/network.go | 60 +++--------------------------------------- 1 file changed, 4 insertions(+), 56 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 132d8d1368..1063ecc33d 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -479,8 +479,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} var ( - nwInfo network.NetworkInfo - networkID string + nwInfo network.NetworkInfo ) if nwCfg.MultiTenancy { @@ -589,13 +588,13 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { for _, ifInfo := range ipamAddResult.interfaceInfo { // TODO: hopefully I can get natInfo here? natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) + networkID, _ := plugin.getNetworkID(args.Netns, &ifInfo, nwCfg) createEpInfoOpt := createEpInfoOpt{ nwCfg: nwCfg, cnsNetworkConfig: ifInfo.NCResponse, ipamAddResult: ipamAddResult, azIpamResult: azIpamResult, args: args, - nwInfo: &nwInfo, // TODO: this is just like a placeholder right? policies: policies, endpointID: endpointID, k8sPodName: k8sPodName, @@ -603,7 +602,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { enableInfraVnet: enableInfraVnet, enableSnatForDNS: enableSnatForDNS, natInfo: natInfo, - networkID: networkID, // TODO: is this the right place to get the network ID, and if so, it will never change! + networkID: networkID, ifInfo: &ifInfo, ipamAddConfig: &ipamAddConfig, @@ -614,15 +613,9 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return err } - //ep, err = plugin.createEndpoint(epInfo, nwCfg.CNSUrl) if err != nil { return err } - // TODO: is this how we choose which ep to use? - // the endpoint that is saved in the statefile is represented fully by the ep info/ep with the InfraNIC type - if ifInfo.NICType == cns.InfraNIC { - //saveEp = ep - } epInfos = append(epInfos, epInfo) // TODO: should this statement be based on the current iteration instead of the constant ifIndex? sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", @@ -635,50 +628,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } plugin.nm.EndpointCreate(cnsclient, epInfos) - // // Create network - // if nwInfoErr != nil { - // // Network does not exist. - // logger.Info("Creating network", zap.String("networkID", networkID)) - // sendEvent(plugin, fmt.Sprintf("[cni-net] Creating network %v.", networkID)) - // // opts map needs to get passed in here - // if nwInfo, err = plugin.createNetworkInternal(networkID, policies, ipamAddConfig, ipamAddResult, ifIndex); err != nil { - // logger.Error("Create network failed", zap.Error(err)) - // return err - // } - // logger.Info("Created network", - // zap.String("networkId", networkID), - // zap.String("subnet", ipamAddResult.hostSubnetPrefix.String())) - // sendEvent(plugin, fmt.Sprintf("[cni-net] Created network %v with subnet %v.", networkID, ipamAddResult.hostSubnetPrefix.String())) - // } - - // natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) - - // createEndpointInternalOpt := createEndpointInternalOpt{ - // nwCfg: nwCfg, - // cnsNetworkConfig: ipamAddResult.interfaceInfo[0].NCResponse, // Alex will fix this in the for loop - // ipamAddResult: ipamAddResult, - // azIpamResult: azIpamResult, - // args: args, - // nwInfo: &nwInfo, - // policies: policies, - // endpointID: endpointID, - // k8sPodName: k8sPodName, - // k8sNamespace: k8sNamespace, - // enableInfraVnet: enableInfraVnet, - // enableSnatForDNS: enableSnatForDNS, - // natInfo: natInfo, - // } - - // var epInfo network.EndpointInfo - // epInfo, err = plugin.createEndpointInternal(&createEndpointInternalOpt, ifIndex) - // if err != nil { - // logger.Error("Endpoint creation failed", zap.Error(err)) - // return err - // } - - // sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", - // ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) - return nil } @@ -704,7 +653,6 @@ type createEpInfoOpt struct { ipamAddResult IPAMAddResult azIpamResult *cniTypesCurr.Result args *cniSkel.CmdArgs - nwInfo *network.NetworkInfo policies []policy.Policy endpointID string k8sPodName string @@ -785,7 +733,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn } nwInfo = network.NetworkInfo{ - Id: "azure-" + opt.ifInfo.MacAddress.String(), + Id: opt.networkID, Mode: opt.ipamAddConfig.nwCfg.Mode, MasterIfName: masterIfName, AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, From 75c194dfed812abc22b8b58a0cc7b6994401ee88 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 1 Apr 2024 17:32:02 -0700 Subject: [PATCH 022/102] unify creation of nw and endpoint info by removing switch --- cni/network/network.go | 307 ++++++++++++++++++----------------------- 1 file changed, 134 insertions(+), 173 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 1063ecc33d..41b43d46d4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -647,6 +647,18 @@ After both sections are done, we can call createNetworkInternal (pass in epInfo) Will need to modify those two functions as code from createEndpointInternal, for example, was pulled out into this function below */ // maybe create a createEndpointOpt struct for all the things we need to pass in + +func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string { + switch opt.ifInfo.NICType { + case cns.InfraNIC: + return plugin.findMasterInterfaceBySubnet(opt.ipamAddConfig.nwCfg, &opt.ifInfo.HostSubnetPrefix) + case cns.DelegatedVMNIC: + return plugin.findInterfaceByMAC(opt.ifInfo.MacAddress.String()) + default: + return "" + } +} + type createEpInfoOpt struct { nwCfg *cni.NetworkConfig cnsNetworkConfig *cns.GetNetworkContainerResponse @@ -674,195 +686,144 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn ) // populate endpoint info section - switch opt.ifInfo.NICType { - case cns.InfraNIC: - masterIfName := plugin.findMasterInterfaceBySubnet(opt.ipamAddConfig.nwCfg, &opt.ifInfo.HostSubnetPrefix) - if masterIfName == "" { - err := plugin.Errorf("Failed to find the master interface") - return nil, err - } - logger.Info("Found master interface", zap.String("ifname", masterIfName)) - if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { - return nil, err - } - - nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) - if err != nil { - return nil, err - } - - nwInfo = network.NetworkInfo{ - Id: opt.networkID, - Mode: opt.ipamAddConfig.nwCfg.Mode, - MasterIfName: masterIfName, - AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, - BridgeName: opt.ipamAddConfig.nwCfg.Bridge, - EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above - DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT TODO: Check what takes precedence - Policies: opt.policies, - NetNs: opt.ipamAddConfig.args.Netns, - Options: opt.ipamAddConfig.options, - DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // TODO: check if IPV6Mode field can be deprecated // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above - IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, - ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above - IsIPv6Enabled: opt.ipv6Enabled, - NICType: string(cns.InfraNIC), - } - - if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { - logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) - return nil, err - } - setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) - - case cns.DelegatedVMNIC: - masterIfName := plugin.findInterfaceByMAC(opt.ifInfo.MacAddress.String()) - if masterIfName == "" { - err := plugin.Errorf("Failed to find the master interface") - return nil, err - } - logger.Info("Found master interface", zap.String("ifname", masterIfName)) - if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { - return nil, err - } - - nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) - if err != nil { - return nil, err - } - - nwInfo = network.NetworkInfo{ - Id: opt.networkID, - Mode: opt.ipamAddConfig.nwCfg.Mode, - MasterIfName: masterIfName, - AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, - BridgeName: opt.ipamAddConfig.nwCfg.Bridge, - EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: see above - DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT: see above - NetNs: opt.ipamAddConfig.args.Netns, - Options: opt.ipamAddConfig.options, - DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: see above - NICType: string(cns.DelegatedVMNIC), - } + masterIfName := plugin.findMasterInterface(opt) + if masterIfName == "" { + err := plugin.Errorf("Failed to find the master interface") + return nil, err + } + logger.Info("Found master interface", zap.String("ifname", masterIfName)) + if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { + return nil, err + } - if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { - logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) - return nil, err - } - setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) + nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) + if err != nil { + return nil, err + } + nwInfo = network.NetworkInfo{ + Id: opt.networkID, + Mode: opt.ipamAddConfig.nwCfg.Mode, + MasterIfName: masterIfName, + AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, + BridgeName: opt.ipamAddConfig.nwCfg.Bridge, + EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT TODO: Check what takes precedence + Policies: opt.policies, // not present in non-infra + NetNs: opt.ipamAddConfig.args.Netns, + Options: opt.ipamAddConfig.options, + DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, + IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // not present in non-infra TODO: check if IPV6Mode field can be deprecated // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, // not present in non-infra + ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + IsIPv6Enabled: opt.ipv6Enabled, // not present in non-infra + NICType: string(opt.ifInfo.NICType), + } + + if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { + logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) + return nil, err } + setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) // populate endpoint info // taken from createEndpointInternal function // populate endpoint info fields code is below // did not move cns client code because we don't need it here - switch opt.ifInfo.NICType { - case cns.DelegatedVMNIC: - // secondary - var addresses []net.IPNet - for _, ipconfig := range opt.ifInfo.IPConfigs { - addresses = append(addresses, ipconfig.Address) - } - - epInfo = network.EndpointInfo{ - ContainerID: epInfo.ContainerID, - NetNsPath: epInfo.NetNsPath, - IPAddresses: addresses, - Routes: opt.ifInfo.Routes, - MacAddress: opt.ifInfo.MacAddress, - NICType: opt.ifInfo.NICType, - SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, - } - case cns.BackendNIC: - // todo - case cns.InfraNIC: - // InfraNic - // taken from create endpoint internal function - // TODO: does this change the behavior (we moved the infra nic code into here rather than above the switch statement)? (it shouldn't) - // at this point you are 100% certain that the interface passed in is the infra nic, and thus, the default interface info - defaultInterfaceInfo := opt.ifInfo - epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultInterfaceInfo.DNS, opt.k8sNamespace) - if err != nil { - err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) - return nil, err - } - policyArgs := PolicyArgs{ - // pass podsubnet info etc. part of epinfo - nwInfo: &nwInfo, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? getEndpointPolicies requires nwInfo.Subnets only (checked) - nwCfg: opt.nwCfg, - ipconfigs: defaultInterfaceInfo.IPConfigs, - } - endpointPolicies, err := getEndpointPolicies(policyArgs) - if err != nil { - logger.Error("Failed to get endpoint policies", zap.Error(err)) - return nil, err - } - - opt.policies = append(opt.policies, endpointPolicies...) - - vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) - if opt.nwCfg.Mode != OpModeTransparent { - // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. - // IT will result in unpredictable behavior if API server decides to - // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", nwInfo.Id, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) The dummy nwInfo has the Id, right? - } - epInfo = network.EndpointInfo{ - Id: opt.endpointID, - ContainerID: opt.args.ContainerID, - NetNsPath: opt.args.Netns, - IfName: opt.args.IfName, - Data: make(map[string]interface{}), - EndpointDNS: epDNSInfo, - Policies: opt.policies, - IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, - EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, - EnableMultiTenancy: opt.nwCfg.MultiTenancy, - EnableInfraVnet: opt.enableInfraVnet, - EnableSnatForDns: opt.enableSnatForDNS, - PODName: opt.k8sPodName, - PODNameSpace: opt.k8sNamespace, - SkipHotAttachEp: false, // Hot attach at the time of endpoint creation - IPV6Mode: opt.nwCfg.IPV6Mode, - VnetCidrs: opt.nwCfg.VnetCidrs, - ServiceCidrs: opt.nwCfg.ServiceCidrs, - NATInfo: opt.natInfo, - NICType: cns.InfraNIC, - SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, // we know we are the default interface at this point - Routes: defaultInterfaceInfo.Routes, - } - - epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) - if err != nil { - logger.Error("failed to get policies from runtime configurations", zap.Error(err)) - return nil, plugin.Errorf(err.Error()) - } - epInfo.Policies = append(epInfo.Policies, epPolicies...) + // taken from create endpoint internal function + // TODO: does this change the behavior (we moved the infra nic code into here rather than above the switch statement)? (it shouldn't) + // at this point you are 100% certain that the interface passed in is the infra nic, and thus, the default interface info + defaultInterfaceInfo := opt.ifInfo + epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultInterfaceInfo.DNS, opt.k8sNamespace) // Probably won't panic if given bad values + if err != nil { + // TODO: will it error out if we have a secondary endpoint that has blank DNS though? If so, problem! + err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) + return nil, err + } + policyArgs := PolicyArgs{ + // pass podsubnet info etc. part of epinfo + nwInfo: &nwInfo, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? getEndpointPolicies requires nwInfo.Subnets only (checked) + nwCfg: opt.nwCfg, + ipconfigs: defaultInterfaceInfo.IPConfigs, + } + endpointPolicies, err := getEndpointPolicies(policyArgs) + if err != nil { + // TODO: should only error out if we have an ip config and it is not readable + logger.Error("Failed to get endpoint policies", zap.Error(err)) + return nil, err + } - // Populate addresses. - for _, ipconfig := range defaultInterfaceInfo.IPConfigs { - epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) - } + opt.policies = append(opt.policies, endpointPolicies...) + + vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) + if opt.nwCfg.Mode != OpModeTransparent { + // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. + // IT will result in unpredictable behavior if API server decides to + // reorder DELETE and ADD call for new incarnation of same POD. + vethName = fmt.Sprintf("%s%s%s", nwInfo.Id, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) The dummy nwInfo has the Id, right? + } + + // for secondary + var addresses []net.IPNet + for _, ipconfig := range opt.ifInfo.IPConfigs { + addresses = append(addresses, ipconfig.Address) + } + + epInfo = network.EndpointInfo{ + Id: opt.endpointID, + ContainerID: opt.args.ContainerID, + NetNsPath: opt.args.Netns, + IfName: opt.args.IfName, + Data: make(map[string]interface{}), + EndpointDNS: epDNSInfo, + Policies: opt.policies, + IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, + EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, + EnableMultiTenancy: opt.nwCfg.MultiTenancy, + EnableInfraVnet: opt.enableInfraVnet, + EnableSnatForDns: opt.enableSnatForDNS, + PODName: opt.k8sPodName, + PODNameSpace: opt.k8sNamespace, + SkipHotAttachEp: false, // Hot attach at the time of endpoint creation + IPV6Mode: opt.nwCfg.IPV6Mode, + VnetCidrs: opt.nwCfg.VnetCidrs, + ServiceCidrs: opt.nwCfg.ServiceCidrs, + NATInfo: opt.natInfo, + NICType: opt.ifInfo.NICType, + SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, // we know we are the default interface at this point + Routes: defaultInterfaceInfo.Routes, + // added the following for delegated vm nic + IPAddresses: addresses, + MacAddress: opt.ifInfo.MacAddress, + } + + epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) // not specific to delegated or infra + if err != nil { + logger.Error("failed to get policies from runtime configurations", zap.Error(err)) + return nil, plugin.Errorf(err.Error()) + } + epInfo.Policies = append(epInfo.Policies, epPolicies...) - if opt.ipamAddResult.ipv6Enabled { - epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working - } + // Populate addresses. + for _, ipconfig := range defaultInterfaceInfo.IPConfigs { + epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) + } - if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { - epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address - } + if opt.ipamAddResult.ipv6Enabled { // not specific to this particular interface + epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working + } - if opt.nwCfg.MultiTenancy { - plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, defaultInterfaceInfo) - } + if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { + epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address + } - setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) + if opt.nwCfg.MultiTenancy { + plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, defaultInterfaceInfo) } + setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) + // populate endpoint info with network info fields // TODO: maybe instead of creating a network info, directly modify epInfo, but some helper methods rely on networkinfo... epInfo.MasterIfName = nwInfo.MasterIfName From 69f78f721405f54f56611aaca4a56e9451cadf8d Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 1 Apr 2024 21:58:26 -0700 Subject: [PATCH 023/102] change functions to consume ep info instead of nw info --- network/bridge_networkclient_linux.go | 16 ++++---- network/manager.go | 6 +-- network/manager_mock.go | 10 ++--- network/network.go | 48 +++++++---------------- network/network_linux.go | 56 +++++++++++++-------------- 5 files changed, 58 insertions(+), 78 deletions(-) diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go index 48e3941701..c66cffce3d 100644 --- a/network/bridge_networkclient_linux.go +++ b/network/bridge_networkclient_linux.go @@ -25,7 +25,7 @@ func newErrorLinuxBridgeClient(errStr string) error { type LinuxBridgeClient struct { bridgeName string hostInterfaceName string - nwInfo NetworkInfo + epInfo EndpointInfo netlink netlink.NetlinkInterface nuClient networkutils.NetworkUtils } @@ -33,13 +33,13 @@ type LinuxBridgeClient struct { func NewLinuxBridgeClient( bridgeName string, hostInterfaceName string, - nwInfo NetworkInfo, + epInfo EndpointInfo, nl netlink.NetlinkInterface, plc platform.ExecClient, ) *LinuxBridgeClient { client := &LinuxBridgeClient{ bridgeName: bridgeName, - nwInfo: nwInfo, + epInfo: epInfo, hostInterfaceName: hostInterfaceName, netlink: nl, nuClient: networkutils.NewNetworkUtils(nl, plc), @@ -112,9 +112,9 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error { return err } - if client.nwInfo.IPV6Mode != "" { + if client.epInfo.IPV6Mode != "" { // for ipv6 node cidr set broute accept - if err := ebtables.SetBrouteAcceptByCidr(&client.nwInfo.Subnets[1].Prefix, ebtables.IPV6, ebtables.Append, ebtables.Accept); err != nil { + if err := ebtables.SetBrouteAcceptByCidr(&client.epInfo.Subnets[1].Prefix, ebtables.IPV6, ebtables.Append, ebtables.Accept); err != nil { return err } @@ -137,7 +137,7 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error { } // Enable VEPA for host policy enforcement if necessary. - if client.nwInfo.Mode == opModeTunnel { + if client.epInfo.Mode == opModeTunnel { logger.Info("Enabling VEPA mode for", zap.String("hostInterfaceName", client.hostInterfaceName)) if err := ebtables.SetVepaMode(client.bridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Append); err != nil { return err @@ -152,7 +152,7 @@ func (client *LinuxBridgeClient) DeleteL2Rules(extIf *externalInterface) { ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete) ebtables.SetArpReply(extIf.IPAddresses[0].IP, extIf.MacAddress, ebtables.Delete) ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete) - if client.nwInfo.IPV6Mode != "" { + if client.epInfo.IPV6Mode != "" { if len(extIf.IPAddresses) > 1 { ebtables.SetBrouteAcceptByCidr(extIf.IPAddresses[1], ebtables.IPV6, ebtables.Delete, ebtables.Accept) } @@ -180,7 +180,7 @@ func (client *LinuxBridgeClient) SetHairpinOnHostInterface(enable bool) error { } func (client *LinuxBridgeClient) setBrouteRedirect(action string) error { - if client.nwInfo.ServiceCidrs != "" { + if client.epInfo.ServiceCidrs != "" { if err := ebtables.SetBrouteAcceptByCidr(nil, ebtables.IPV4, ebtables.Append, ebtables.RedirectAccept); err != nil { return err } diff --git a/network/manager.go b/network/manager.go index 3527e41511..605dfc7c8f 100644 --- a/network/manager.go +++ b/network/manager.go @@ -91,7 +91,7 @@ type NetworkManager interface { AddExternalInterface(ifName string, subnet string) error - CreateNetwork(nwInfo *NetworkInfo) error + CreateNetwork(nwInfo *EndpointInfo) error DeleteNetwork(networkID string) error GetNetworkInfo(networkID string) (NetworkInfo, error) // FindNetworkIDFromNetNs returns the network name that contains an endpoint created for this netNS, errNetworkNotFound if no network is found @@ -305,11 +305,11 @@ func (nm *networkManager) AddExternalInterface(ifName string, subnet string) err } // CreateNetwork creates a new container network. -func (nm *networkManager) CreateNetwork(nwInfo *NetworkInfo) error { +func (nm *networkManager) CreateNetwork(epInfo *EndpointInfo) error { nm.Lock() defer nm.Unlock() - _, err := nm.newNetwork(nwInfo) + _, err := nm.newNetwork(epInfo) if err != nil { return err } diff --git a/network/manager_mock.go b/network/manager_mock.go index 14b5df3281..9f25a1ee1a 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -6,7 +6,7 @@ import ( // MockNetworkManager is a mock structure for Network Manager type MockNetworkManager struct { - TestNetworkInfoMap map[string]*NetworkInfo + TestNetworkInfoMap map[string]*EndpointInfo TestEndpointInfoMap map[string]*EndpointInfo TestEndpointClient *MockEndpointClient } @@ -14,7 +14,7 @@ type MockNetworkManager struct { // NewMockNetworkmanager returns a new mock func NewMockNetworkmanager(mockEndpointclient *MockEndpointClient) *MockNetworkManager { return &MockNetworkManager{ - TestNetworkInfoMap: make(map[string]*NetworkInfo), + TestNetworkInfoMap: make(map[string]*EndpointInfo), TestEndpointInfoMap: make(map[string]*EndpointInfo), TestEndpointClient: mockEndpointclient, } @@ -34,7 +34,7 @@ func (nm *MockNetworkManager) AddExternalInterface(ifName string, subnet string) } // CreateNetwork mock -func (nm *MockNetworkManager) CreateNetwork(nwInfo *NetworkInfo) error { +func (nm *MockNetworkManager) CreateNetwork(nwInfo *EndpointInfo) error { nm.TestNetworkInfoMap[nwInfo.Id] = nwInfo return nil } @@ -45,11 +45,11 @@ func (nm *MockNetworkManager) DeleteNetwork(networkID string) error { } // GetNetworkInfo mock -func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (NetworkInfo, error) { +func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (EndpointInfo, error) { if info, exists := nm.TestNetworkInfoMap[networkID]; exists { return *info, nil } - return NetworkInfo{}, errNetworkNotFound + return EndpointInfo{}, errNetworkNotFound } // CreateEndpoint mock diff --git a/network/network.go b/network/network.go index 74b4e29182..226d39a5be 100644 --- a/network/network.go +++ b/network/network.go @@ -162,29 +162,29 @@ func (nm *networkManager) findExternalInterfaceByName(ifName string) *externalIn } // NewNetwork creates a new container network. -func (nm *networkManager) newNetwork(nwInfo *NetworkInfo) (*network, error) { +func (nm *networkManager) newNetwork(epInfo *EndpointInfo) (*network, error) { var nw *network var err error - logger.Info("Creating", zap.String("network", nwInfo.PrettyString())) + logger.Info("Creating", zap.String("network", epInfo.PrettyString())) defer func() { if err != nil { - logger.Error("Failed to create network", zap.String("id", nwInfo.Id), zap.Error(err)) + logger.Error("Failed to create network", zap.String("id", epInfo.Id), zap.Error(err)) } }() // Set defaults. - if nwInfo.Mode == "" { - nwInfo.Mode = opModeDefault + if epInfo.Mode == "" { + epInfo.Mode = opModeDefault } // If the master interface name is provided, find the external interface by name // else use subnet to to find the interface var extIf *externalInterface - if len(strings.TrimSpace(nwInfo.MasterIfName)) > 0 { - extIf = nm.findExternalInterfaceByName(nwInfo.MasterIfName) + if len(strings.TrimSpace(epInfo.MasterIfName)) > 0 { + extIf = nm.findExternalInterfaceByName(epInfo.MasterIfName) } else { - extIf = nm.findExternalInterfaceBySubnet(nwInfo.Subnets[0].Prefix.String()) + extIf = nm.findExternalInterfaceBySubnet(epInfo.Subnets[0].Prefix.String()) } if extIf == nil { err = errSubnetNotFound @@ -192,22 +192,22 @@ func (nm *networkManager) newNetwork(nwInfo *NetworkInfo) (*network, error) { } // Make sure this network does not already exist. - if extIf.Networks[nwInfo.Id] != nil { + if extIf.Networks[epInfo.Id] != nil { err = errNetworkExists return nil, err } // Call the OS-specific implementation. - nw, err = nm.newNetworkImpl(nwInfo, extIf) + nw, err = nm.newNetworkImpl(epInfo, extIf) if err != nil { return nil, err } // Add the network object. - nw.Subnets = nwInfo.Subnets - extIf.Networks[nwInfo.Id] = nw + nw.Subnets = epInfo.Subnets + extIf.Networks[epInfo.Id] = nw - logger.Info("Created network on interface", zap.String("id", nwInfo.Id), zap.String("Name", extIf.Name)) + logger.Info("Created network on interface", zap.String("id", epInfo.Id), zap.String("Name", extIf.Name)) return nw, nil } @@ -312,27 +312,7 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo if nwGetErr != nil { // Create the network if it is not found // populate network info with fields from ep info - nwInfo = NetworkInfo{ - MasterIfName: epInfo.MasterIfName, - AdapterName: epInfo.AdapterName, - Id: epInfo.NetworkId, - Mode: epInfo.Mode, - Subnets: epInfo.Subnets, - PodSubnet: epInfo.PodSubnet, - DNS: epInfo.NetworkDNS, - Policies: epInfo.Policies, - BridgeName: epInfo.BridgeName, - EnableSnatOnHost: epInfo.EnableSnatOnHost, - NetNs: epInfo.NetNs, - Options: epInfo.Options, - DisableHairpinOnHostInterface: epInfo.DisableHairpinOnHostInterface, - IPV6Mode: epInfo.IPV6Mode, - IPAMType: epInfo.IPAMType, - ServiceCidrs: epInfo.ServiceCidrs, - IsIPv6Enabled: epInfo.IsIPv6Enabled, - NICType: string(epInfo.NICType), - } - err := nm.CreateNetwork(&nwInfo) + err := nm.CreateNetwork(epInfo) if err != nil { // TODO: error messages/handling are different in this file // err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) diff --git a/network/network_linux.go b/network/network_linux.go index 47f68b420d..5ccb9df65c 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -59,22 +59,22 @@ func newErrorNetworkManager(errStr string) error { type route netlink.Route // NewNetworkImpl creates a new container network. -func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { +func (nm *networkManager) newNetworkImpl(epInfo *EndpointInfo, extIf *externalInterface) (*network, error) { // Connect the external interface. var ( vlanid int ifName string ) - opt, _ := nwInfo.Options[genericData].(map[string]interface{}) - logger.Info("opt options", zap.Any("opt", opt), zap.Any("options", nwInfo.Options)) + opt, _ := epInfo.Options[genericData].(map[string]interface{}) + logger.Info("opt options", zap.Any("opt", opt), zap.Any("options", epInfo.Options)) - switch nwInfo.Mode { + switch epInfo.Mode { case opModeTunnel: fallthrough case opModeBridge: logger.Info("create bridge") ifName = extIf.BridgeName - if err := nm.connectExternalInterface(extIf, nwInfo); err != nil { + if err := nm.connectExternalInterface(extIf, epInfo); err != nil { return nil, err } @@ -84,7 +84,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt case opModeTransparent: logger.Info("Transparent mode") ifName = extIf.Name - if nwInfo.IPV6Mode != "" { + if epInfo.IPV6Mode != "" { nu := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) if err := nu.EnableIPV6Forwarding(); err != nil { return nil, fmt.Errorf("Ipv6 forwarding failed: %w", err) @@ -111,7 +111,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nil, errNetworkModeInvalid } - err := nm.handleCommonOptions(ifName, nwInfo) + err := nm.handleCommonOptions(ifName, epInfo) if err != nil { logger.Error("handleCommonOptions failed with", zap.Error(err)) return nil, err @@ -119,28 +119,28 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt // Create the network object. nw := &network{ - Id: nwInfo.Id, - Mode: nwInfo.Mode, + Id: epInfo.Id, + Mode: epInfo.Mode, Endpoints: make(map[string]*endpoint), extIf: extIf, VlanId: vlanid, - DNS: nwInfo.DNS, - EnableSnatOnHost: nwInfo.EnableSnatOnHost, + DNS: epInfo.NetworkDNS, + EnableSnatOnHost: epInfo.EnableSnatOnHost, } return nw, nil } -func (nm *networkManager) handleCommonOptions(ifName string, nwInfo *NetworkInfo) error { +func (nm *networkManager) handleCommonOptions(ifName string, epInfo *EndpointInfo) error { var err error - if routes, exists := nwInfo.Options[RoutesKey]; exists { + if routes, exists := epInfo.Options[RoutesKey]; exists { err = addRoutes(nm.netlink, nm.netio, ifName, routes.([]RouteInfo)) if err != nil { return err } } - if iptcmds, exists := nwInfo.Options[IPTablesKey]; exists { + if iptcmds, exists := epInfo.Options[IPTablesKey]; exists { err = nm.addToIptables(iptcmds.([]iptables.IPTableEntry)) if err != nil { return err @@ -157,7 +157,7 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error { if nw.VlanId != 0 { networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, ovsctl.NewOvsctl(), nm.netlink, nm.plClient) } else { - networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, NetworkInfo{}, nm.netlink, nm.plClient) + networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, EndpointInfo{}, nm.netlink, nm.plClient) } // Disconnect the interface if this was the last network using it. @@ -463,7 +463,7 @@ func (nm *networkManager) applyDNSConfig(extIf *externalInterface, ifName string } // ConnectExternalInterface connects the given host interface to a bridge. -func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error { +func (nm *networkManager) connectExternalInterface(extIf *externalInterface, epInfo *EndpointInfo) error { var ( err error networkClient NetworkClient @@ -486,16 +486,16 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI } // If a bridge name is not specified, generate one based on the external interface index. - bridgeName := nwInfo.BridgeName + bridgeName := epInfo.BridgeName if bridgeName == "" { bridgeName = fmt.Sprintf("%s%d", bridgePrefix, hostIf.Index) } - opt, _ := nwInfo.Options[genericData].(map[string]interface{}) + opt, _ := epInfo.Options[genericData].(map[string]interface{}) if opt != nil && opt[VlanIDKey] != nil { networkClient = NewOVSClient(bridgeName, extIf.Name, ovsctl.NewOvsctl(), nm.netlink, nm.plClient) } else { - networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *nwInfo, nm.netlink, nm.plClient) + networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *epInfo, nm.netlink, nm.plClient) } // Check if the bridge already exists. @@ -582,7 +582,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI } // External interface hairpin on. - if !nwInfo.DisableHairpinOnHostInterface { + if !epInfo.DisableHairpinOnHostInterface { logger.Info("Setting link hairpin on", zap.String("Name", hostIf.Name)) if err = networkClient.SetHairpinOnHostInterface(true); err != nil { return err @@ -607,14 +607,14 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI logger.Info("Applied dns config on", zap.Any("DNSInfo", extIf.DNSInfo), zap.String("bridgeName", bridgeName)) } - if nwInfo.IPV6Mode == IPV6Nat { + if epInfo.IPV6Mode == IPV6Nat { // adds pod cidr gateway ip to bridge - if err = nm.addIpv6NatGateway(nwInfo); err != nil { + if err = nm.addIpv6NatGateway(epInfo); err != nil { logger.Error("Adding IPv6 Nat Gateway failed with", zap.Error(err)) return err } - if err = nm.addIpv6SnatRule(extIf, nwInfo); err != nil { + if err = nm.addIpv6SnatRule(extIf, epInfo); err != nil { logger.Error("Adding IPv6 Snat Rule failed with", zap.Error(err)) return err } @@ -674,16 +674,16 @@ func (nm *networkManager) addToIptables(cmds []iptables.IPTableEntry) error { } // Add ipv6 nat gateway IP on bridge -func (nm *networkManager) addIpv6NatGateway(nwInfo *NetworkInfo) error { +func (nm *networkManager) addIpv6NatGateway(epInfo *EndpointInfo) error { logger.Info("Adding ipv6 nat gateway on azure bridge") - for _, subnetInfo := range nwInfo.Subnets { + for _, subnetInfo := range epInfo.Subnets { if subnetInfo.Family == platform.AfINET6 { ipAddr := []net.IPNet{{ IP: subnetInfo.Gateway, Mask: subnetInfo.Prefix.Mask, }} nuc := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) - err := nuc.AssignIPToInterface(nwInfo.BridgeName, ipAddr) + err := nuc.AssignIPToInterface(epInfo.BridgeName, ipAddr) if err != nil { return newErrorNetworkManager(err.Error()) } @@ -694,13 +694,13 @@ func (nm *networkManager) addIpv6NatGateway(nwInfo *NetworkInfo) error { } // snat ipv6 traffic to secondary ipv6 ip before leaving VM -func (nm *networkManager) addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { +func (nm *networkManager) addIpv6SnatRule(extIf *externalInterface, epInfo *EndpointInfo) error { var ( ipv6SnatRuleSet bool ipv6SubnetPrefix net.IPNet ) - for _, subnet := range nwInfo.Subnets { + for _, subnet := range epInfo.Subnets { if subnet.Family == platform.AfINET6 { ipv6SubnetPrefix = subnet.Prefix break From c790e8b9d6423a32d9192525b93aa4a80adf45bd Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 1 Apr 2024 21:59:55 -0700 Subject: [PATCH 024/102] remove unused variable accidentally added earlier --- network/endpoint.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index 04dceae050..0936b5fd92 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -113,8 +113,6 @@ type EndpointInfo struct { // ServiceCidrs omitted IsIPv6Enabled bool // NICType omitted - - IfInfo *InterfaceInfo // for secondary interfaces when creating endpoint struct } // RouteInfo contains information about an IP route. From d137832af7be1889518f8047af06dc23f8b34209 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 2 Apr 2024 09:57:29 -0700 Subject: [PATCH 025/102] update old azure ipam invoker to use ep info and change ids to network ids when appropriate previously we renamed the NetworkInfo symbol to EndpointInfo in lots of places, but the Id in Network Info is NOT the same as Endpoint Info, so while the code compiles, code that previously used the id field of the network info struct would now be using the id field of the endpoint info struct. It should use the NetworkId field of the endpoint info struct instead. --- cni/network/invoker_azure.go | 4 ++-- cni/network/network.go | 14 ++++++-------- network/manager.go | 12 ++++++------ network/manager_mock.go | 4 ++-- network/network.go | 10 +++++----- network/network_linux.go | 4 ++-- 6 files changed, 23 insertions(+), 25 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index b891ab9dd1..67332ff933 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -29,7 +29,7 @@ const ( type AzureIPAMInvoker struct { plugin delegatePlugin - nwInfo *network.NetworkInfo + nwInfo *network.EndpointInfo } type delegatePlugin interface { @@ -39,7 +39,7 @@ type delegatePlugin interface { } // Create an IPAM instance every time a CNI action is called. -func NewAzureIpamInvoker(plugin *NetPlugin, nwInfo *network.NetworkInfo) *AzureIPAMInvoker { +func NewAzureIpamInvoker(plugin *NetPlugin, nwInfo *network.EndpointInfo) *AzureIPAMInvoker { return &AzureIPAMInvoker{ plugin: plugin, nwInfo: nwInfo, diff --git a/cni/network/network.go b/cni/network/network.go index 41b43d46d4..2cb9d5ce78 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -344,10 +344,11 @@ func (plugin *NetPlugin) getNetworkID(netNs string, interfaceInfo *network.Inter return networkID, nil } -// get network info -func (plugin *NetPlugin) getNetworkInfo(netNs string, interfaceInfo *network.InterfaceInfo, nwCfg *cni.NetworkConfig) network.NetworkInfo { +// get network info for legacy +func (plugin *NetPlugin) getNetworkInfo(netNs string, interfaceInfo *network.InterfaceInfo, nwCfg *cni.NetworkConfig) network.EndpointInfo { networkID, _ := plugin.getNetworkID(netNs, interfaceInfo, nwCfg) nwInfo, _ := plugin.nm.GetNetworkInfo(networkID) + return nwInfo } @@ -478,10 +479,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { options := make(map[string]any) ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} - var ( - nwInfo network.NetworkInfo - ) - if nwCfg.MultiTenancy { plugin.report.Context = "AzureCNIMultitenancy" plugin.multitenancyClient.Init(cnsClient, AzureNetIOShim{}) @@ -514,7 +511,8 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { case network.AzureCNS: plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) default: - nwInfo = plugin.getNetworkInfo(args.Netns, nil, nwCfg) + // legacy + nwInfo := plugin.getNetworkInfo(args.Netns, nil, nwCfg) plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) } } @@ -1221,7 +1219,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { k8sPodName string k8sNamespace string networkID string - nwInfo network.NetworkInfo + nwInfo network.EndpointInfo epInfo *network.EndpointInfo cniMetric telemetry.AIMetric ) diff --git a/network/manager.go b/network/manager.go index 605dfc7c8f..8f1983aa25 100644 --- a/network/manager.go +++ b/network/manager.go @@ -93,7 +93,7 @@ type NetworkManager interface { CreateNetwork(nwInfo *EndpointInfo) error DeleteNetwork(networkID string) error - GetNetworkInfo(networkID string) (NetworkInfo, error) + GetNetworkInfo(networkID string) (EndpointInfo, error) // FindNetworkIDFromNetNs returns the network name that contains an endpoint created for this netNS, errNetworkNotFound if no network is found FindNetworkIDFromNetNs(netNs string) (string, error) GetNumEndpointsByContainerID(containerID string) int @@ -336,21 +336,21 @@ func (nm *networkManager) DeleteNetwork(networkID string) error { } // GetNetworkInfo returns information about the given network. -func (nm *networkManager) GetNetworkInfo(networkId string) (NetworkInfo, error) { +func (nm *networkManager) GetNetworkInfo(networkId string) (EndpointInfo, error) { nm.Lock() defer nm.Unlock() nw, err := nm.getNetwork(networkId) if err != nil { - return NetworkInfo{}, err + return EndpointInfo{}, err } - nwInfo := NetworkInfo{ - Id: networkId, + nwInfo := EndpointInfo{ + NetworkId: networkId, Subnets: nw.Subnets, Mode: nw.Mode, EnableSnatOnHost: nw.EnableSnatOnHost, - DNS: nw.DNS, + NetworkDNS: nw.DNS, Options: make(map[string]interface{}), } diff --git a/network/manager_mock.go b/network/manager_mock.go index 9f25a1ee1a..8ab0740501 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -35,7 +35,7 @@ func (nm *MockNetworkManager) AddExternalInterface(ifName string, subnet string) // CreateNetwork mock func (nm *MockNetworkManager) CreateNetwork(nwInfo *EndpointInfo) error { - nm.TestNetworkInfoMap[nwInfo.Id] = nwInfo + nm.TestNetworkInfoMap[nwInfo.NetworkId] = nwInfo return nil } @@ -147,7 +147,7 @@ func (nm *MockNetworkManager) GetNumEndpointsByContainerID(_ string) int { numEndpoints := 0 for _, network := range nm.TestNetworkInfoMap { - if _, err := nm.GetAllEndpoints(network.Id); err == nil { + if _, err := nm.GetAllEndpoints(network.NetworkId); err == nil { numEndpoints++ } } diff --git a/network/network.go b/network/network.go index 226d39a5be..f0197357d2 100644 --- a/network/network.go +++ b/network/network.go @@ -169,7 +169,7 @@ func (nm *networkManager) newNetwork(epInfo *EndpointInfo) (*network, error) { logger.Info("Creating", zap.String("network", epInfo.PrettyString())) defer func() { if err != nil { - logger.Error("Failed to create network", zap.String("id", epInfo.Id), zap.Error(err)) + logger.Error("Failed to create network", zap.String("id", epInfo.NetworkId), zap.Error(err)) } }() @@ -192,7 +192,7 @@ func (nm *networkManager) newNetwork(epInfo *EndpointInfo) (*network, error) { } // Make sure this network does not already exist. - if extIf.Networks[epInfo.Id] != nil { + if extIf.Networks[epInfo.NetworkId] != nil { err = errNetworkExists return nil, err } @@ -205,9 +205,9 @@ func (nm *networkManager) newNetwork(epInfo *EndpointInfo) (*network, error) { // Add the network object. nw.Subnets = epInfo.Subnets - extIf.Networks[epInfo.Id] = nw + extIf.Networks[epInfo.NetworkId] = nw - logger.Info("Created network on interface", zap.String("id", epInfo.Id), zap.String("Name", extIf.Name)) + logger.Info("Created network on interface", zap.String("id", epInfo.NetworkId), zap.String("Name", extIf.Name)) return nw, nil } @@ -326,7 +326,7 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo // TODO: pass in network id, or get network id from ep info-- it makes more sense if it came from epInfo // since the network is directly related to the epInfo // TODO: okay to pass in a slice of epInfo this way and just say epIndex is 0, or is refactoring create endpoint needed? - ep, err := nm.CreateEndpoint(cnsclient, nwInfo.Id, epInfo) + ep, err := nm.CreateEndpoint(cnsclient, nwInfo.NetworkId, epInfo) if err != nil { // err = plugin.Errorf("Failed to create endpoint: %v", err) return err //added diff --git a/network/network_linux.go b/network/network_linux.go index 5ccb9df65c..75ff1486c4 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -119,7 +119,7 @@ func (nm *networkManager) newNetworkImpl(epInfo *EndpointInfo, extIf *externalIn // Create the network object. nw := &network{ - Id: epInfo.Id, + Id: epInfo.NetworkId, Mode: epInfo.Mode, Endpoints: make(map[string]*endpoint), extIf: extIf, @@ -731,7 +731,7 @@ func (nm *networkManager) addIpv6SnatRule(extIf *externalInterface, epInfo *Endp return nil } -func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) { +func getNetworkInfoImpl(nwInfo *EndpointInfo, nw *network) { if nw.VlanId != 0 { vlanMap := make(map[string]interface{}) vlanMap[VlanIDKey] = strconv.Itoa(nw.VlanId) From 9e0f0294c376ce1cd341691a629fae16f259a434 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 2 Apr 2024 10:31:29 -0700 Subject: [PATCH 026/102] rename endpoint info id field to EndpointID to remove ambiguity --- cni/network/network.go | 8 ++++---- cni/network/network_test.go | 14 ++++++------- cnm/network/network.go | 2 +- network/endpoint.go | 16 +++++++-------- network/endpoint_linux.go | 16 +++++++-------- network/endpoint_snatroute_linux.go | 4 ++-- network/endpoint_test.go | 32 ++++++++++++++--------------- network/manager.go | 6 +++--- network/manager_mock.go | 2 +- network/mock_endpointclient.go | 4 ++-- network/ovs_endpointclient_linux.go | 2 +- 11 files changed, 53 insertions(+), 53 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 2cb9d5ce78..54e2207894 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -183,11 +183,11 @@ func (plugin *NetPlugin) GetAllEndpointState(networkid string) (*api.AzureCNISta } for _, ep := range eps { - id := ep.Id + id := ep.EndpointID info := api.PodNetworkInterfaceInfo{ PodName: ep.PODName, PodNamespace: ep.PODNameSpace, - PodEndpointId: ep.Id, + PodEndpointId: ep.EndpointID, ContainerID: ep.ContainerID, IPAddresses: ep.IPAddresses, } @@ -769,7 +769,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn } epInfo = network.EndpointInfo{ - Id: opt.endpointID, + EndpointID: opt.endpointID, ContainerID: opt.args.ContainerID, NetNsPath: opt.args.Netns, IfName: opt.args.IfName, @@ -1591,7 +1591,7 @@ func (plugin *NetPlugin) Update(args *cniSkel.CmdArgs) error { // Update the endpoint. logger.Info("Now updating existing endpoint with targetNetworkConfig", - zap.String("endpoint", existingEpInfo.Id), + zap.String("endpoint", existingEpInfo.EndpointID), zap.Any("config", targetNetworkConfig)) if err = plugin.nm.UpdateEndpoint(networkID, existingEpInfo, targetEpInfo); err != nil { err = plugin.Errorf("Failed to update endpoint: %v", err) diff --git a/cni/network/network_test.go b/cni/network/network_test.go index a8f87e36be..2ab6fc6788 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -1024,7 +1024,7 @@ func getTestEndpoint(podname, podnamespace, ipwithcidr, podinterfaceid, infracon ep := acnnetwork.EndpointInfo{ PODName: podname, PODNameSpace: podnamespace, - Id: podinterfaceid, + EndpointID: podinterfaceid, ContainerID: infracontainerid, IPAddresses: []net.IPNet{ *ipnet, @@ -1056,22 +1056,22 @@ func TestGetAllEndpointState(t *testing.T) { res := &api.AzureCNIState{ ContainerInterfaces: map[string]api.PodNetworkInterfaceInfo{ - ep1.Id: { - PodEndpointId: ep1.Id, + ep1.EndpointID: { + PodEndpointId: ep1.EndpointID, PodName: ep1.PODName, PodNamespace: ep1.PODNameSpace, ContainerID: ep1.ContainerID, IPAddresses: ep1.IPAddresses, }, - ep2.Id: { - PodEndpointId: ep2.Id, + ep2.EndpointID: { + PodEndpointId: ep2.EndpointID, PodName: ep2.PODName, PodNamespace: ep2.PODNameSpace, ContainerID: ep2.ContainerID, IPAddresses: ep2.IPAddresses, }, - ep3.Id: { - PodEndpointId: ep3.Id, + ep3.EndpointID: { + PodEndpointId: ep3.EndpointID, PodName: ep3.PODName, PodNamespace: ep3.PODNameSpace, ContainerID: ep3.ContainerID, diff --git a/cnm/network/network.go b/cnm/network/network.go index 078edf76d4..8a286927c4 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -236,7 +236,7 @@ func (plugin *netPlugin) createEndpoint(w http.ResponseWriter, r *http.Request) } epInfo := network.EndpointInfo{ - Id: req.EndpointID, + EndpointID: req.EndpointID, IPAddresses: []net.IPNet{*ipv4Address}, SkipHotAttachEp: true, // Skip hot attach endpoint as it's done in Join } diff --git a/network/endpoint.go b/network/endpoint.go index 0936b5fd92..077e5ef40e 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -59,7 +59,7 @@ type endpoint struct { // EndpointInfo contains read-only information about an endpoint. type EndpointInfo struct { - Id string + EndpointID string ContainerID string NetNsPath string IfName string @@ -152,7 +152,7 @@ type apipaClient interface { func (epInfo *EndpointInfo) PrettyString() string { return fmt.Sprintf("Id:%s ContainerID:%s NetNsPath:%s IfName:%s IfIndex:%d MacAddr:%s IPAddrs:%v Gateways:%v Data:%+v", - epInfo.Id, epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName, epInfo.IfIndex, epInfo.MacAddress.String(), epInfo.IPAddresses, + epInfo.EndpointID, epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName, epInfo.IfIndex, epInfo.MacAddress.String(), epInfo.IPAddresses, epInfo.Gateways, epInfo.Data) } @@ -171,7 +171,7 @@ func (nw *network) newEndpoint( defer func() { if err != nil { - logger.Error("Failed to create endpoint with err", zap.String("id", epInfo.Id), zap.Error(err)) + logger.Error("Failed to create endpoint with err", zap.String("id", epInfo.EndpointID), zap.Error(err)) } }() @@ -270,7 +270,7 @@ func podNameMatches(source string, actualValue string, doExactMatch bool) bool { // GetInfo returns information about the endpoint. func (ep *endpoint) getInfo() *EndpointInfo { info := &EndpointInfo{ - Id: ep.Id, + EndpointID: ep.Id, IPAddresses: ep.IPAddresses, InfraVnetIP: ep.InfraVnetIP, Data: make(map[string]interface{}), @@ -337,13 +337,13 @@ func (nm *networkManager) updateEndpoint(nw *network, exsitingEpInfo *EndpointIn zap.String("id", nw.Id), zap.Any("targetEpInfo", targetEpInfo)) defer func() { if err != nil { - logger.Error("Failed to update endpoint with err", zap.String("id", exsitingEpInfo.Id), zap.Error(err)) + logger.Error("Failed to update endpoint with err", zap.String("id", exsitingEpInfo.EndpointID), zap.Error(err)) } }() - logger.Info("Trying to retrieve endpoint id", zap.String("id", exsitingEpInfo.Id)) + logger.Info("Trying to retrieve endpoint id", zap.String("id", exsitingEpInfo.EndpointID)) - ep := nw.Endpoints[exsitingEpInfo.Id] + ep := nw.Endpoints[exsitingEpInfo.EndpointID] if ep == nil { return errEndpointNotFound } @@ -357,7 +357,7 @@ func (nm *networkManager) updateEndpoint(nw *network, exsitingEpInfo *EndpointIn } // Update routes for existing endpoint - nw.Endpoints[exsitingEpInfo.Id].Routes = ep.Routes + nw.Endpoints[exsitingEpInfo.EndpointID].Routes = ep.Routes return nil } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index ffe9742f23..826a9386d5 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -69,7 +69,7 @@ func (nw *network) newEndpointImpl( containerIf *net.Interface ) - if nw.Endpoints[defaultEpInfo.Id] != nil { + if nw.Endpoints[defaultEpInfo.EndpointID] != nil { logger.Info("[net] Endpoint already exists.") err = errEndpointExists return nil, err @@ -94,12 +94,12 @@ func (nw *network) newEndpointImpl( } else { // Create a veth pair. logger.Info("Generate veth name based on endpoint id") - hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, defaultEpInfo.Id[:7]) - contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, defaultEpInfo.Id[:7]) + hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, defaultEpInfo.EndpointID[:7]) + contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, defaultEpInfo.EndpointID[:7]) } ep := &endpoint{ - Id: defaultEpInfo.Id, + Id: defaultEpInfo.EndpointID, IfName: contIfName, // container veth pair name. In cnm, we won't rename this and docker expects veth name. HostIfName: hostIfName, InfraVnetIP: defaultEpInfo.InfraVnetIP, @@ -394,8 +394,8 @@ func deleteRoutes(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, i func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *EndpointInfo, targetEpInfo *EndpointInfo) (*endpoint, error) { var ep *endpoint - existingEpFromRepository := nw.Endpoints[existingEpInfo.Id] - logger.Info("[updateEndpointImpl] Going to retrieve endpoint with Id to update", zap.String("id", existingEpInfo.Id)) + existingEpFromRepository := nw.Endpoints[existingEpInfo.EndpointID] + logger.Info("[updateEndpointImpl] Going to retrieve endpoint with Id to update", zap.String("id", existingEpInfo.EndpointID)) if existingEpFromRepository == nil { logger.Info("[updateEndpointImpl] Endpoint cannot be updated as it does not exist") return nil, errEndpointNotFound @@ -426,7 +426,7 @@ func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *Endpoi } }() } else { - logger.Info("[updateEndpointImpl] Endpoint cannot be updated as the network namespace does not exist: Epid", zap.String("id", existingEpInfo.Id), + logger.Info("[updateEndpointImpl] Endpoint cannot be updated as the network namespace does not exist: Epid", zap.String("id", existingEpInfo.EndpointID), zap.String("component", "updateEndpointImpl")) return nil, errNamespaceNotFound } @@ -438,7 +438,7 @@ func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *Endpoi // Create the endpoint object. ep = &endpoint{ - Id: existingEpInfo.Id, + Id: existingEpInfo.EndpointID, } // Update existing endpoint state with the new routes to persist diff --git a/network/endpoint_snatroute_linux.go b/network/endpoint_snatroute_linux.go index 0f8de4e1ac..c139393e98 100644 --- a/network/endpoint_snatroute_linux.go +++ b/network/endpoint_snatroute_linux.go @@ -11,11 +11,11 @@ import ( ) func GetSnatHostIfName(epInfo *EndpointInfo) string { - return fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) + return fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.EndpointID[:7]) } func GetSnatContIfName(epInfo *EndpointInfo) string { - return fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) + return fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.EndpointID[:7]) } func AddSnatEndpoint(snatClient *snat.Client) error { diff --git a/network/endpoint_test.go b/network/endpoint_test.go index b7e0d8fe47..3de6cf58fd 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -175,9 +175,9 @@ var _ = Describe("Test Endpoint", func() { Endpoints: map[string]*endpoint{}, } epInfo := &EndpointInfo{ - Id: "768e8deb-eth1", - Data: make(map[string]interface{}), - IfName: eth0IfName, + EndpointID: "768e8deb-eth1", + Data: make(map[string]interface{}), + IfName: eth0IfName, } epInfo.Data[VlanIDKey] = 100 @@ -187,7 +187,7 @@ var _ = Describe("Test Endpoint", func() { netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) - Expect(ep.Id).To(Equal(epInfo.Id)) + Expect(ep.Id).To(Equal(epInfo.EndpointID)) Expect(ep.Gateways).To(BeEmpty()) }) It("should have fields set", func() { @@ -199,7 +199,7 @@ var _ = Describe("Test Endpoint", func() { netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) - Expect(ep.Id).To(Equal(epInfo.Id)) + Expect(ep.Id).To(Equal(epInfo.EndpointID)) Expect(ep.Gateways).NotTo(BeNil()) Expect(len(ep.Gateways)).To(Equal(1)) Expect(ep.Gateways[0].String()).To(Equal("192.168.0.1")) @@ -242,9 +242,9 @@ var _ = Describe("Test Endpoint", func() { Endpoints: map[string]*endpoint{}, } epInfo := &EndpointInfo{ - Id: "768e8deb-eth1", - IfName: eth0IfName, - NICType: cns.InfraNIC, + EndpointID: "768e8deb-eth1", + IfName: eth0IfName, + NICType: cns.InfraNIC, } ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), netio.NewMockNetIO(false, 0), NewMockEndpointClient(func(ep *EndpointInfo) error { @@ -260,7 +260,7 @@ var _ = Describe("Test Endpoint", func() { netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) - Expect(ep.Id).To(Equal(epInfo.Id)) + Expect(ep.Id).To(Equal(epInfo.EndpointID)) }) }) Context("When secondary endpoint client is used", func() { @@ -271,9 +271,9 @@ var _ = Describe("Test Endpoint", func() { extIf: &externalInterface{BridgeName: "testbridge"}, } epInfo := &EndpointInfo{ - Id: "768e8deb-eth1", - IfName: eth0IfName, - NICType: cns.InfraNIC, + EndpointID: "768e8deb-eth1", + IfName: eth0IfName, + NICType: cns.InfraNIC, } secondaryEpInfo := &EndpointInfo{ NICType: cns.DelegatedVMNIC, @@ -292,7 +292,7 @@ var _ = Describe("Test Endpoint", func() { ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) Expect(err).ToNot(HaveOccurred()) - Expect(ep.Id).To(Equal(epInfo.Id)) + Expect(ep.Id).To(Equal(epInfo.EndpointID)) }) It("Should add endpoint when there are no errors", func() { @@ -300,7 +300,7 @@ var _ = Describe("Test Endpoint", func() { ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) Expect(err).ToNot(HaveOccurred()) - Expect(ep.Id).To(Equal(epInfo.Id)) + Expect(ep.Id).To(Equal(epInfo.EndpointID)) }) }) }) @@ -312,8 +312,8 @@ var _ = Describe("Test Endpoint", func() { nw := &network{} existingEpInfo := &EndpointInfo{ - Id: "768e8deb-eth1", - IfName: eth0IfName, + EndpointID: "768e8deb-eth1", + IfName: eth0IfName, } targetEpInfo := &EndpointInfo{} err := nm.updateEndpoint(nw, existingEpInfo, targetEpInfo) diff --git a/network/manager.go b/network/manager.go index 8f1983aa25..94b4637e5c 100644 --- a/network/manager.go +++ b/network/manager.go @@ -427,7 +427,7 @@ func (nm *networkManager) GetEndpointState(networkID, endpointID string) (*Endpo return nil, errors.Wrapf(err, "Get endpoint API returend with error") } epInfo := &EndpointInfo{ - Id: endpointID, + EndpointID: endpointID, IfIndex: EndpointIfIndex, // Azure CNI supports only one interface IfName: endpointResponse.EndpointInfo.HostVethName, ContainerID: endpointID, @@ -491,7 +491,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint } ep := &endpoint{ - Id: epInfo.Id, + Id: epInfo.EndpointID, HnsId: epInfo.HNSEndpointID, HostIfName: epInfo.IfName, LocalIP: "", @@ -500,7 +500,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint AllowInboundFromNCToHost: false, EnableSnatOnHost: false, EnableMultitenancy: false, - NetworkContainerID: epInfo.Id, + NetworkContainerID: epInfo.EndpointID, } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) return nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) diff --git a/network/manager_mock.go b/network/manager_mock.go index 8ab0740501..4654cc5929 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -60,7 +60,7 @@ func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfo *En return nil, err } - nm.TestEndpointInfoMap[epInfo.Id] = epInfo + nm.TestEndpointInfoMap[epInfo.EndpointID] = epInfo return &endpoint{}, nil } diff --git a/network/mock_endpointclient.go b/network/mock_endpointclient.go index b95ab47890..b73546a583 100644 --- a/network/mock_endpointclient.go +++ b/network/mock_endpointclient.go @@ -36,11 +36,11 @@ func NewMockEndpointClient(fn func(*EndpointInfo) error) *MockEndpointClient { } func (client *MockEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - if ok := client.endpoints[epInfo.Id]; ok && epInfo.IfName == eth0IfName { + if ok := client.endpoints[epInfo.EndpointID]; ok && epInfo.IfName == eth0IfName { return NewErrorMockEndpointClient("Endpoint already exists") } - client.endpoints[epInfo.Id] = true + client.endpoints[epInfo.EndpointID] = true return client.testAddEndpointFn(epInfo) } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 569232eb9a..97f4186b7d 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -74,7 +74,7 @@ func NewOVSEndpointClient( netioshim: &netio.NetIO{}, } - NewInfraVnetClient(client, epInfo.Id[:7]) + NewInfraVnetClient(client, epInfo.EndpointID[:7]) client.NewSnatClient(nw.SnatBridgeIP, localIP, epInfo) return client From ed3153212f4cc5f974653f6ac33319b9b82d200b Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 2 Apr 2024 11:13:32 -0700 Subject: [PATCH 027/102] change nw info to ep info in windows --- network/endpoint_windows.go | 2 +- network/endpoint_windows_test.go | 10 +++++----- network/network_windows.go | 26 +++++++++++++------------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 200682e994..586c8237bf 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -193,7 +193,7 @@ func (nw *network) addIPv6NeighborEntryForGateway(epInfo *EndpointInfo, plc plat // run powershell cmd to set neighbor entry for gw ip to 12-34-56-78-9a-bc cmd := fmt.Sprintf("New-NetNeighbor -IPAddress %s -InterfaceAlias \"%s (%s)\" -LinkLayerAddress \"%s\"", - nw.Subnets[1].Gateway.String(), containerIfNamePrefix, epInfo.Id, defaultGwMac) + nw.Subnets[1].Gateway.String(), containerIfNamePrefix, epInfo.EndpointID, defaultGwMac) if out, err = plc.ExecutePowershellCommand(cmd); err != nil { logger.Error("Adding ipv6 gw neigh entry failed", zap.Any("out", out), zap.Error(err)) diff --git a/network/endpoint_windows_test.go b/network/endpoint_windows_test.go index a352b7316f..aa59a62c3a 100644 --- a/network/endpoint_windows_test.go +++ b/network/endpoint_windows_test.go @@ -27,7 +27,7 @@ func TestNewAndDeleteEndpointImplHnsV2(t *testing.T) { } epInfo := &EndpointInfo{ - Id: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", + EndpointID: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", ContainerID: "545055c2-1462-42c8-b222-e75d0b291632", NetNsPath: "fakeNameSpace", IfName: "eth0", @@ -71,7 +71,7 @@ func TestNewEndpointImplHnsv2Timesout(t *testing.T) { } epInfo := &EndpointInfo{ - Id: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", + EndpointID: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", ContainerID: "545055c2-1462-42c8-b222-e75d0b291632", NetNsPath: "fakeNameSpace", IfName: "eth0", @@ -98,7 +98,7 @@ func TestDeleteEndpointImplHnsv2Timeout(t *testing.T) { Hnsv2 = hnswrapper.NewHnsv2wrapperFake() epInfo := &EndpointInfo{ - Id: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", + EndpointID: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", ContainerID: "545055c2-1462-42c8-b222-e75d0b291632", NetNsPath: "fakeNameSpace", IfName: "eth0", @@ -147,7 +147,7 @@ func TestCreateEndpointImplHnsv1Timeout(t *testing.T) { } epInfo := &EndpointInfo{ - Id: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", + EndpointID: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", ContainerID: "545055c2-1462-42c8-b222-e75d0b291632", NetNsPath: "fakeNameSpace", IfName: "eth0", @@ -174,7 +174,7 @@ func TestDeleteEndpointImplHnsv1Timeout(t *testing.T) { Hnsv1 = hnswrapper.NewHnsv1wrapperFake() epInfo := &EndpointInfo{ - Id: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", + EndpointID: "753d3fb6-e9b3-49e2-a109-2acc5dda61f1", ContainerID: "545055c2-1462-42c8-b222-e75d0b291632", NetNsPath: "fakeNameSpace", IfName: "eth0", diff --git a/network/network_windows.go b/network/network_windows.go index 29917db239..913bf029b6 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -86,7 +86,7 @@ func EnableHnsV1Timeout(timeoutValue int) { } // newNetworkImplHnsV1 creates a new container network for HNSv1. -func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { +func (nm *networkManager) newNetworkImplHnsV1(nwInfo *EndpointInfo, extIf *externalInterface) (*network, error) { var ( vlanid int err error @@ -111,9 +111,9 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern // Initialize HNS network. hnsNetwork := &hcsshim.HNSNetwork{ - Name: nwInfo.Id, + Name: nwInfo.NetworkId, NetworkAdapterName: networkAdapterName, - DNSServerList: strings.Join(nwInfo.DNS.Servers, ","), + DNSServerList: strings.Join(nwInfo.NetworkDNS.Servers, ","), Policies: policy.SerializePolicies(policy.NetworkPolicy, nwInfo.Policies, nil, false, false), } @@ -172,7 +172,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern // Create the network object. nw := &network{ - Id: nwInfo.Id, + Id: nwInfo.NetworkId, HnsId: hnsResponse.Id, Mode: nwInfo.Mode, Endpoints: make(map[string]*endpoint), @@ -193,7 +193,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern return nw, nil } -func (nm *networkManager) appIPV6RouteEntry(nwInfo *NetworkInfo) error { +func (nm *networkManager) appIPV6RouteEntry(nwInfo *EndpointInfo) error { var ( err error out string @@ -227,13 +227,13 @@ func (nm *networkManager) appIPV6RouteEntry(nwInfo *NetworkInfo) error { } // configureHcnEndpoint configures hcn endpoint for creation -func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *externalInterface) (*hcn.HostComputeNetwork, error) { +func (nm *networkManager) configureHcnNetwork(nwInfo *EndpointInfo, extIf *externalInterface) (*hcn.HostComputeNetwork, error) { // Initialize HNS network. hcnNetwork := &hcn.HostComputeNetwork{ - Name: nwInfo.Id, + Name: nwInfo.NetworkId, Dns: hcn.Dns{ - Domain: nwInfo.DNS.Suffix, - ServerList: nwInfo.DNS.Servers, + Domain: nwInfo.NetworkDNS.Suffix, + ServerList: nwInfo.NetworkDNS.Servers, }, Ipams: []hcn.Ipam{ { @@ -352,7 +352,7 @@ func (nm *networkManager) addIPv6DefaultRoute() error { } // newNetworkImplHnsV2 creates a new container network for HNSv2. -func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { +func (nm *networkManager) newNetworkImplHnsV2(nwInfo *EndpointInfo, extIf *externalInterface) (*network, error) { hcnNetwork, err := nm.configureHcnNetwork(nwInfo, extIf) if err != nil { logger.Error("Failed to configure hcn network due to", zap.Error(err)) @@ -400,7 +400,7 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern // Create the network object. nw := &network{ - Id: nwInfo.Id, + Id: nwInfo.NetworkId, HnsId: hnsResponse.Id, Mode: nwInfo.Mode, Endpoints: make(map[string]*endpoint), @@ -414,7 +414,7 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern } // NewNetworkImpl creates a new container network. -func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { +func (nm *networkManager) newNetworkImpl(nwInfo *EndpointInfo, extIf *externalInterface) (*network, error) { if useHnsV2, err := UseHnsV2(nwInfo.NetNs); useHnsV2 { if err != nil { return nil, err @@ -463,5 +463,5 @@ func (nm *networkManager) deleteNetworkImplHnsV2(nw *network) error { return err } -func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) { +func getNetworkInfoImpl(nwInfo *EndpointInfo, nw *network) { } From c4f2cbd8d1c6b0f8d3644ad35f5f69883afa15b1 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 2 Apr 2024 11:42:02 -0700 Subject: [PATCH 028/102] adjust comments --- cni/network/network.go | 281 +---------------------------------------- network/network.go | 3 +- 2 files changed, 4 insertions(+), 280 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 54e2207894..5f4bde6f9d 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -535,42 +535,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) - // if !nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS { - // nwInfo, nwInfoErr = plugin.getNetworkInfo(args.Netns, ipamAddResult, nwCfg) - // } - - // Check whether the network already exists. - - // Handle consecutive ADD calls for infrastructure containers. - // This is a temporary work around for issue #57253 of Kubernetes. - // We can delete this if statement once they fix it. - // Issue link: https://github.com/kubernetes/kubernetes/issues/57253 - - // if nwInfoErr == nil { - // logger.Info("Found network with subnet", - // zap.String("network", networkID), - // zap.String("subnet", nwInfo.Subnets[0].Prefix.String())) - // nwInfo.IPAMType = nwCfg.IPAM.Type - // options = nwInfo.Options - - // var resultSecondAdd *cniTypesCurr.Result - // resultSecondAdd, err = plugin.handleConsecutiveAdd(args, endpointID, networkID, &nwInfo, nwCfg) - // if err != nil { - // logger.Error("handleConsecutiveAdd failed", zap.Error(err)) - // return err - // } - - // if resultSecondAdd != nil { - // ifIndex, err = findDefaultInterface(ipamAddResult) - // if err != nil { - // ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{}) - // ifIndex = len(ipamAddResult.interfaceInfo) - 1 - // } - // ipamAddResult.interfaceInfo[ifIndex] = convertCniResultToInterfaceInfo(resultSecondAdd) - // return nil - // } - // } - defer func() { //nolint:gocritic if err != nil { // for multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips @@ -580,8 +544,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } }() - // TODO: is it possible nwInfo is not populated? YES it seems so, in which case the nwInfo is an empty struct! - // TODO: This will mean when we try to do nwInfo.Id or nwInfo.Subnets in createEpInfo, we'll have problems! epInfos := []*network.EndpointInfo{} for _, ifInfo := range ipamAddResult.interfaceInfo { // TODO: hopefully I can get natInfo here? @@ -630,22 +592,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } -// loop over each interface info and call createEndpoint with it -// you don't have access to a list of interface infos here -// no need for default index because we only trigger the default index code when the type is the infra nic -/** -Overview: -Create endpoint info -Pass pointer to endpoint info to a new function to populate endpoint info fields (right now we don't separate the populating code into functions) -Pass pointer to endpoint info to another new function to populate network info fields (or we can just do it in one big switch statement) - To populate network info, use Paul's code which should get the network info fields from various things passed in (we should pass those things in here too) - Potentially problematic getting the network id because plugin.getNetworkName seems to just always return nwCfg.Name (NEED ASSISTANCE) - But if we do get the network id correctly, just pass to Paul's code to get the network info fields which we can populate epInfo with -After both sections are done, we can call createNetworkInternal (pass in epInfo) and createEndpointInternal (pass in epInfo) - Will need to modify those two functions as code from createEndpointInternal, for example, was pulled out into this function below -*/ -// maybe create a createEndpointOpt struct for all the things we need to pass in - func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string { switch opt.ifInfo.NICType { case cns.InfraNIC: @@ -706,7 +652,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, BridgeName: opt.ipamAddConfig.nwCfg.Bridge, EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above - DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT TODO: Check what takes precedence + DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT (resolved by making nw and ep dns infos) Policies: opt.policies, // not present in non-infra NetNs: opt.ipamAddConfig.args.Netns, Options: opt.ipamAddConfig.options, @@ -725,13 +671,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) // populate endpoint info - // taken from createEndpointInternal function // populate endpoint info fields code is below - // did not move cns client code because we don't need it here - - // taken from create endpoint internal function - // TODO: does this change the behavior (we moved the infra nic code into here rather than above the switch statement)? (it shouldn't) - // at this point you are 100% certain that the interface passed in is the infra nic, and thus, the default interface info defaultInterfaceInfo := opt.ifInfo epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultInterfaceInfo.DNS, opt.k8sNamespace) // Probably won't panic if given bad values if err != nil { @@ -759,7 +699,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. // IT will result in unpredictable behavior if API server decides to // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", nwInfo.Id, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) The dummy nwInfo has the Id, right? + vethName = fmt.Sprintf("%s%s%s", nwInfo.Id, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) We use the nwInfo we generated above } // for secondary @@ -789,7 +729,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn ServiceCidrs: opt.nwCfg.ServiceCidrs, NATInfo: opt.natInfo, NICType: opt.ifInfo.NICType, - SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, // we know we are the default interface at this point + SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, Routes: defaultInterfaceInfo.Routes, // added the following for delegated vm nic IPAddresses: addresses, @@ -823,7 +763,6 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) // populate endpoint info with network info fields - // TODO: maybe instead of creating a network info, directly modify epInfo, but some helper methods rely on networkinfo... epInfo.MasterIfName = nwInfo.MasterIfName epInfo.AdapterName = nwInfo.AdapterName epInfo.NetworkId = nwInfo.Id @@ -879,73 +818,6 @@ func (plugin *NetPlugin) getNetworkDNSSettings(nwCfg *cni.NetworkConfig, dns net return nwDNSInfo, nil } -// func (plugin *NetPlugin) createNetworkInternal( -// networkID string, -// policies []policy.Policy, -// ipamAddConfig IPAMAddConfig, -// ipamAddResult IPAMAddResult, -// ifIndex int, -// ) (network.NetworkInfo, error) { -// nwInfo := network.NetworkInfo{} -// ipamAddResult.hostSubnetPrefix.IP = ipamAddResult.hostSubnetPrefix.IP.Mask(ipamAddResult.hostSubnetPrefix.Mask) -// ipamAddConfig.nwCfg.IPAM.Subnet = ipamAddResult.hostSubnetPrefix.String() -// // Find the master interface. -// masterIfName := plugin.findMasterInterface(ipamAddConfig.nwCfg, &ipamAddResult.hostSubnetPrefix) -// if masterIfName == "" { -// err := plugin.Errorf("Failed to find the master interface") -// return nwInfo, err -// } -// logger.Info("Found master interface", zap.String("ifname", masterIfName)) - -// // Add the master as an external interface. -// err := plugin.nm.AddExternalInterface(masterIfName, ipamAddResult.hostSubnetPrefix.String()) -// if err != nil { -// err = plugin.Errorf("Failed to add external interface: %v", err) -// return nwInfo, err -// } - -// nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.interfaceInfo[ifIndex].DNS) -// if err != nil { -// err = plugin.Errorf("Failed to getDNSSettings: %v", err) -// return nwInfo, err -// } - -// logger.Info("DNS Info", zap.Any("info", nwDNSInfo)) - -// // Create the network. -// nwInfo = network.NetworkInfo{ -// Id: networkID, -// Mode: ipamAddConfig.nwCfg.Mode, -// MasterIfName: masterIfName, -// AdapterName: ipamAddConfig.nwCfg.AdapterName, -// BridgeName: ipamAddConfig.nwCfg.Bridge, -// EnableSnatOnHost: ipamAddConfig.nwCfg.EnableSnatOnHost, -// DNS: nwDNSInfo, -// Policies: policies, -// NetNs: ipamAddConfig.args.Netns, -// Options: ipamAddConfig.options, -// DisableHairpinOnHostInterface: ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, -// IPV6Mode: ipamAddConfig.nwCfg.IPV6Mode, // TODO: check if IPV6Mode field can be deprecated -// IPAMType: ipamAddConfig.nwCfg.IPAM.Type, -// ServiceCidrs: ipamAddConfig.nwCfg.ServiceCidrs, -// IsIPv6Enabled: ipamAddResult.ipv6Enabled, -// } - -// if err = addSubnetToNetworkInfo(ipamAddResult, &nwInfo, ifIndex); err != nil { -// logger.Info("Failed to add subnets to networkInfo", -// zap.Error(err)) -// return nwInfo, err -// } -// setNetworkOptions(ipamAddResult.interfaceInfo[0].NCResponse, &nwInfo) // Alex will fix this - -// err = plugin.nm.CreateNetwork(&nwInfo) -// if err != nil { -// err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) -// } - -// return nwInfo, err -// } - // construct network info with ipv4/ipv6 subnets func addSubnetToNetworkInfo(interfaceInfo network.InterfaceInfo, nwInfo *network.NetworkInfo) error { for _, ipConfig := range interfaceInfo.IPConfigs { @@ -969,153 +841,6 @@ func addSubnetToNetworkInfo(interfaceInfo network.InterfaceInfo, nwInfo *network return nil } -// type createEndpointInternalOpt struct { -// nwCfg *cni.NetworkConfig -// cnsNetworkConfig *cns.GetNetworkContainerResponse -// ipamAddResult IPAMAddResult -// azIpamResult *cniTypesCurr.Result -// args *cniSkel.CmdArgs -// nwInfo *network.NetworkInfo -// policies []policy.Policy -// endpointID string -// k8sPodName string -// k8sNamespace string -// enableInfraVnet bool -// enableSnatForDNS bool -// natInfo []policy.NATInfo -// } - -// func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt, ifIndex int) (network.EndpointInfo, error) { -// epInfo := network.EndpointInfo{} - -// ifInfo := opt.ipamAddResult.interfaceInfo[ifIndex] -// epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, ifInfo.DNS, opt.k8sNamespace) -// if err != nil { -// err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) -// return epInfo, err -// } -// policyArgs := PolicyArgs{ -// nwInfo: opt.nwInfo, -// nwCfg: opt.nwCfg, -// ipconfigs: ifInfo.IPConfigs, -// } -// endpointPolicies, err := getEndpointPolicies(policyArgs) -// if err != nil { -// logger.Error("Failed to get endpoint policies", zap.Error(err)) -// return epInfo, err -// } - -// opt.policies = append(opt.policies, endpointPolicies...) - -// vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) -// if opt.nwCfg.Mode != OpModeTransparent { -// // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. -// // IT will result in unpredictable behavior if API server decides to -// // reorder DELETE and ADD call for new incarnation of same POD. -// vethName = fmt.Sprintf("%s%s%s", opt.nwInfo.Id, opt.args.ContainerID, opt.args.IfName) -// } - -// epInfo = network.EndpointInfo{ -// Id: opt.endpointID, -// ContainerID: opt.args.ContainerID, -// NetNsPath: opt.args.Netns, -// IfName: opt.args.IfName, -// Data: make(map[string]interface{}), -// DNS: epDNSInfo, -// Policies: opt.policies, -// IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, -// EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, -// EnableMultiTenancy: opt.nwCfg.MultiTenancy, -// EnableInfraVnet: opt.enableInfraVnet, -// EnableSnatForDns: opt.enableSnatForDNS, -// PODName: opt.k8sPodName, -// PODNameSpace: opt.k8sNamespace, -// SkipHotAttachEp: false, // Hot attach at the time of endpoint creation -// IPV6Mode: opt.nwCfg.IPV6Mode, -// VnetCidrs: opt.nwCfg.VnetCidrs, -// ServiceCidrs: opt.nwCfg.ServiceCidrs, -// NATInfo: opt.natInfo, -// NICType: cns.InfraNIC, -// SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[ifIndex].SkipDefaultRoutes, -// Routes: ifInfo.Routes, -// } - -// epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) -// if err != nil { -// logger.Error("failed to get policies from runtime configurations", zap.Error(err)) -// return epInfo, plugin.Errorf(err.Error()) -// } -// epInfo.Policies = append(epInfo.Policies, epPolicies...) - -// // Populate addresses. -// for _, ipconfig := range ifInfo.IPConfigs { -// epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) -// } - -// if opt.ipamAddResult.ipv6Enabled { -// epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working -// } - -// if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { -// epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address -// } - -// if opt.nwCfg.MultiTenancy { -// plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, &ifInfo) -// } - -// setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) - -// cnsclient, err := cnscli.New(opt.nwCfg.CNSUrl, defaultRequestTimeout) -// if err != nil { -// logger.Error("failed to initialized cns client", zap.String("url", opt.nwCfg.CNSUrl), -// zap.String("error", err.Error())) -// return epInfo, plugin.Errorf(err.Error()) -// } - -// epInfos := []*network.EndpointInfo{&epInfo} -// epIndex := 0 // epInfo index for InfraNIC -// // get secondary interface info -// for i := 0; i < len(opt.ipamAddResult.interfaceInfo); i++ { -// switch opt.ipamAddResult.interfaceInfo[i].NICType { -// case cns.DelegatedVMNIC: -// // secondary -// var addresses []net.IPNet -// for _, ipconfig := range opt.ipamAddResult.interfaceInfo[i].IPConfigs { -// addresses = append(addresses, ipconfig.Address) -// } - -// epInfos = append(epInfos, -// &network.EndpointInfo{ -// ContainerID: epInfo.ContainerID, -// NetNsPath: epInfo.NetNsPath, -// IPAddresses: addresses, -// Routes: opt.ipamAddResult.interfaceInfo[i].Routes, -// MacAddress: opt.ipamAddResult.interfaceInfo[i].MacAddress, -// NICType: opt.ipamAddResult.interfaceInfo[i].NICType, -// SkipDefaultRoutes: opt.ipamAddResult.interfaceInfo[i].SkipDefaultRoutes, -// }) -// case cns.BackendNIC: -// // todo -// case cns.InfraNIC: -// epIndex = i -// continue -// default: -// // Error catch for unsupported NICType? -// } -// } - -// // Create the endpoint. -// logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) -// sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) -// err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, epInfos, epIndex) -// if err != nil { -// err = plugin.Errorf("Failed to create endpoint: %v", err) -// } - -// return epInfo, err -// } - // Get handles CNI Get commands. func (plugin *NetPlugin) Get(args *cniSkel.CmdArgs) error { var ( diff --git a/network/network.go b/network/network.go index f0197357d2..9cbd4c4a29 100644 --- a/network/network.go +++ b/network/network.go @@ -57,7 +57,7 @@ type network struct { SnatBridgeIP string } -// NetworkInfo contains read-only information about a container network. +// NetworkInfo contains read-only information about a container network. Use EndpointInfo instead when possible. type NetworkInfo struct { MasterIfName string AdapterName string @@ -325,7 +325,6 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo // sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) // TODO: pass in network id, or get network id from ep info-- it makes more sense if it came from epInfo // since the network is directly related to the epInfo - // TODO: okay to pass in a slice of epInfo this way and just say epIndex is 0, or is refactoring create endpoint needed? ep, err := nm.CreateEndpoint(cnsclient, nwInfo.NetworkId, epInfo) if err != nil { // err = plugin.Errorf("Failed to create endpoint: %v", err) From 811610a2f4541e52c366a8f2671b983f06ab846f Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 2 Apr 2024 14:12:03 -0700 Subject: [PATCH 029/102] move all methods in create ep info dependent on nw info to use ep info instead (windows and linux) addSubnetToNetworkInfo, setNetworkOptions, and getEndpointPolicies. getEndpointPolicies will now take just the subnets needed as a parameter rather than the whole nw or ep info. --- cni/network/network.go | 32 ++++++++++++++++---------------- cni/network/network_linux.go | 2 +- cni/network/network_windows.go | 10 +++++----- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 5f4bde6f9d..a8be26697d 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -79,9 +79,9 @@ type NetPlugin struct { } type PolicyArgs struct { - nwInfo *network.NetworkInfo - nwCfg *cni.NetworkConfig - ipconfigs []*network.IPConfig + subnetInfos []network.SubnetInfo + nwCfg *cni.NetworkConfig + ipconfigs []*network.IPConfig } // client for node network service @@ -626,7 +626,7 @@ type createEpInfoOpt struct { func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointInfo, error) { // you can modify to pass in whatever else you need var ( epInfo network.EndpointInfo - nwInfo network.NetworkInfo + nwInfo network.EndpointInfo ) // populate endpoint info section @@ -644,15 +644,15 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn if err != nil { return nil, err } - - nwInfo = network.NetworkInfo{ - Id: opt.networkID, + // this struct is for organization and represents fields from the network to be merged into the endpoint info later + nwInfo = network.EndpointInfo{ + NetworkId: opt.networkID, Mode: opt.ipamAddConfig.nwCfg.Mode, MasterIfName: masterIfName, AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, BridgeName: opt.ipamAddConfig.nwCfg.Bridge, EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above - DNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT (resolved by making nw and ep dns infos) + NetworkDNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT (resolved by making nw and ep dns infos) Policies: opt.policies, // not present in non-infra NetNs: opt.ipamAddConfig.args.Netns, Options: opt.ipamAddConfig.options, @@ -661,7 +661,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, // not present in non-infra ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above IsIPv6Enabled: opt.ipv6Enabled, // not present in non-infra - NICType: string(opt.ifInfo.NICType), + NICType: opt.ifInfo.NICType, } if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { @@ -681,9 +681,9 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn } policyArgs := PolicyArgs{ // pass podsubnet info etc. part of epinfo - nwInfo: &nwInfo, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? getEndpointPolicies requires nwInfo.Subnets only (checked) - nwCfg: opt.nwCfg, - ipconfigs: defaultInterfaceInfo.IPConfigs, + subnetInfos: nwInfo.Subnets, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? getEndpointPolicies requires nwInfo.Subnets only (checked) + nwCfg: opt.nwCfg, + ipconfigs: defaultInterfaceInfo.IPConfigs, } endpointPolicies, err := getEndpointPolicies(policyArgs) if err != nil { @@ -699,7 +699,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. // IT will result in unpredictable behavior if API server decides to // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", nwInfo.Id, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) We use the nwInfo we generated above + vethName = fmt.Sprintf("%s%s%s", nwInfo.NetworkId, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) We use the nwInfo we generated above } // for secondary @@ -765,7 +765,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // populate endpoint info with network info fields epInfo.MasterIfName = nwInfo.MasterIfName epInfo.AdapterName = nwInfo.AdapterName - epInfo.NetworkId = nwInfo.Id + epInfo.NetworkId = nwInfo.NetworkId epInfo.Mode = nwInfo.Mode epInfo.Subnets = nwInfo.Subnets epInfo.PodSubnet = nwInfo.PodSubnet @@ -775,7 +775,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn epInfo.DisableHairpinOnHostInterface = nwInfo.DisableHairpinOnHostInterface epInfo.IPAMType = nwInfo.IPAMType epInfo.IsIPv6Enabled = nwInfo.IsIPv6Enabled - epInfo.NetworkDNS = nwInfo.DNS + epInfo.NetworkDNS = nwInfo.NetworkDNS // now our ep info should have the full combined information from both the network and endpoint structs return &epInfo, nil @@ -819,7 +819,7 @@ func (plugin *NetPlugin) getNetworkDNSSettings(nwCfg *cni.NetworkConfig, dns net } // construct network info with ipv4/ipv6 subnets -func addSubnetToNetworkInfo(interfaceInfo network.InterfaceInfo, nwInfo *network.NetworkInfo) error { +func addSubnetToNetworkInfo(interfaceInfo network.InterfaceInfo, nwInfo *network.EndpointInfo) error { for _, ipConfig := range interfaceInfo.IPConfigs { ip, podSubnetPrefix, err := net.ParseCIDR(ipConfig.Address.String()) if err != nil { diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index fd0a8417b1..68702ff17f 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -41,7 +41,7 @@ func addSnatForDNS(gwIPString string, epInfo *network.EndpointInfo, result *netw result.Routes = append(result.Routes, network.RouteInfo{Dst: *dnsIPNet, Gw: gwIP}) } -func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { +func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.EndpointInfo) { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { logger.Info("Setting Network Options") vlanMap := make(map[string]interface{}) diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index bf325e438f..21a3d3e43e 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -116,7 +116,7 @@ func addDefaultRoute(_ string, _ *network.EndpointInfo, _ *network.InterfaceInfo func addSnatForDNS(_ string, _ *network.EndpointInfo, _ *network.InterfaceInfo) { } -func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { +func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.EndpointInfo) { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { logger.Info("Setting Network Options") vlanMap := make(map[string]interface{}) @@ -330,7 +330,7 @@ func getEndpointPolicies(args PolicyArgs) ([]policy.Policy, error) { var policies []policy.Policy if args.nwCfg.IPV6Mode == network.IPV6Nat { - ipv6Policy, err := getIPV6EndpointPolicy(args.nwInfo) + ipv6Policy, err := getIPV6EndpointPolicy(args.subnetInfos) if err != nil { return nil, errors.Wrap(err, "failed to get ipv6 endpoint policy") } @@ -373,15 +373,15 @@ func getLoopbackDSRPolicy(args PolicyArgs) ([]policy.Policy, error) { return policies, nil } -func getIPV6EndpointPolicy(nwInfo *network.NetworkInfo) (policy.Policy, error) { +func getIPV6EndpointPolicy(subnetInfos []network.SubnetInfo) (policy.Policy, error) { var eppolicy policy.Policy - if len(nwInfo.Subnets) < 2 { + if len(subnetInfos) < 2 { return eppolicy, fmt.Errorf("network state doesn't have ipv6 subnet") } // Everything should be snat'd except podcidr - exceptionList := []string{nwInfo.Subnets[1].Prefix.String()} + exceptionList := []string{subnetInfos[1].Prefix.String()} rawPolicy, _ := json.Marshal(&hcsshim.OutboundNatPolicy{ Policy: hcsshim.Policy{Type: hcsshim.OutboundNat}, Exceptions: exceptionList, From 50db70f3e1e1ed5cd238c8b6288cd9526a5e13cb Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 2 Apr 2024 14:30:13 -0700 Subject: [PATCH 030/102] make cnm compile (not necessarily correct) --- cnm/network/network.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cnm/network/network.go b/cnm/network/network.go index 8a286927c4..5d7d352d37 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -145,9 +145,9 @@ func (plugin *netPlugin) createNetwork(w http.ResponseWriter, r *http.Request) { } // Process request. - nwInfo := network.NetworkInfo{ - Id: req.NetworkID, - Options: req.Options, + nwInfo := network.EndpointInfo{ + NetworkId: req.NetworkID, + Options: req.Options, } // Parse network options. @@ -247,8 +247,8 @@ func (plugin *netPlugin) createEndpoint(w http.ResponseWriter, r *http.Request) if err != nil { log.Errorf("failed to init CNS client", err) } - // We only pass a single epInfo, so can pass default index as 0 - err = plugin.nm.CreateEndpoint(cnscli, req.NetworkID, []*network.EndpointInfo{&epInfo}, 0) + _, err = plugin.nm.CreateEndpoint(cnscli, req.NetworkID, &epInfo) + // TODO: Because create endpoint no longer assigns to the map or saves to a file, you need to handle it in cnm right here! if err != nil { plugin.SendErrorResponse(w, err) return From 9383597561ee13d08110235e759e894296cdb78a Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 2 Apr 2024 15:18:28 -0700 Subject: [PATCH 031/102] make all tests compile except endpoint test secondary client (windows and linux) (not necessarily correct) --- cni/network/invoker_azure_test.go | 10 +-- cni/network/network_linux_test.go | 4 +- cni/network/network_test.go | 6 +- cni/network/network_windows_test.go | 132 ++++++++++++---------------- network/endpoint_test.go | 12 +-- network/network_test.go | 20 ++--- network/network_windows_test.go | 32 +++---- 7 files changed, 98 insertions(+), 118 deletions(-) diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 1ca9fd43ac..72cff01cc2 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -112,8 +112,8 @@ func getResult(ips ...string) []*network.IPConfig { return res } -func getNwInfo(subnetv4, subnetv6 string) *network.NetworkInfo { - nwinfo := &network.NetworkInfo{} +func getNwInfo(subnetv4, subnetv6 string) *network.EndpointInfo { + nwinfo := &network.EndpointInfo{} if subnetv4 != "" { nwinfo.Subnets = append(nwinfo.Subnets, network.SubnetInfo{ Prefix: *getCIDRNotationForAddress(subnetv4), @@ -131,7 +131,7 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { require := require.New(t) type fields struct { plugin delegatePlugin - nwInfo *network.NetworkInfo + nwInfo *network.EndpointInfo } type args struct { nwCfg *cni.NetworkConfig @@ -259,7 +259,7 @@ func TestAzureIPAMInvoker_Delete(t *testing.T) { require := require.New(t) type fields struct { plugin delegatePlugin - nwInfo *network.NetworkInfo + nwInfo *network.EndpointInfo } type args struct { address *net.IPNet @@ -394,7 +394,7 @@ func TestRemoveIpamState_Add(t *testing.T) { requires := require.New(t) type fields struct { plugin delegatePlugin - nwInfo *network.NetworkInfo + nwInfo *network.EndpointInfo } type args struct { nwCfg *cni.NetworkConfig diff --git a/cni/network/network_linux_test.go b/cni/network/network_linux_test.go index ed2994dbc4..b0fda74548 100644 --- a/cni/network/network_linux_test.go +++ b/cni/network/network_linux_test.go @@ -16,7 +16,7 @@ func TestSetNetworkOptions(t *testing.T) { tests := []struct { name string cnsNwConfig cns.GetNetworkContainerResponse - nwInfo network.NetworkInfo + nwInfo network.EndpointInfo expectedVlanID string expectedSnatBrIP string }{ @@ -34,7 +34,7 @@ func TestSetNetworkOptions(t *testing.T) { GatewayIPAddress: "169.254.0.1", }, }, - nwInfo: network.NetworkInfo{ + nwInfo: network.EndpointInfo{ Options: make(map[string]interface{}), }, expectedVlanID: "1", diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 2ab6fc6788..0423727d77 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -1042,13 +1042,13 @@ func TestGetAllEndpointState(t *testing.T) { ep2 := getTestEndpoint("podname2", "podnamespace2", "10.0.0.2/24", "podinterfaceid2", "testcontainerid2") ep3 := getTestEndpoint("podname3", "podnamespace3", "10.240.1.242/16", "podinterfaceid3", "testcontainerid3") - err := plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep1}, 0) // giving zero to test UT, probably need to change + _, err := plugin.nm.CreateEndpoint(nil, networkid, ep1) require.NoError(t, err) - err = plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep2}, 0) + _, err = plugin.nm.CreateEndpoint(nil, networkid, ep2) require.NoError(t, err) - err = plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep3}, 0) + _, err = plugin.nm.CreateEndpoint(nil, networkid, ep3) require.NoError(t, err) state, err := plugin.GetAllEndpointState(networkid) diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 39a55f83c7..e37a289344 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -31,13 +31,13 @@ func TestAddWithRunTimeNetPolicies(t *testing.T) { tests := []struct { name string - nwInfo network.NetworkInfo + nwInfo network.EndpointInfo wantErr bool wantErrMsg string }{ { name: "add ipv6 endpoint policy", - nwInfo: network.NetworkInfo{ + nwInfo: network.EndpointInfo{ Subnets: []network.SubnetInfo{ { Gateway: net.ParseIP("10.240.0.1"), @@ -56,7 +56,7 @@ func TestAddWithRunTimeNetPolicies(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - p, err := getIPV6EndpointPolicy(&tt.nwInfo) + p, err := getIPV6EndpointPolicy(tt.nwInfo.Subnets) if tt.wantErr { require.Error(t, err) } else { @@ -143,7 +143,7 @@ func TestSetNetworkOptions(t *testing.T) { tests := []struct { name string cnsNwConfig cns.GetNetworkContainerResponse - nwInfo network.NetworkInfo + nwInfo network.EndpointInfo expectedVlanID string }{ { @@ -153,7 +153,7 @@ func TestSetNetworkOptions(t *testing.T) { ID: 1, }, }, - nwInfo: network.NetworkInfo{ + nwInfo: network.EndpointInfo{ Options: make(map[string]interface{}), }, expectedVlanID: "1", @@ -294,7 +294,7 @@ func TestDSRPolciy(t *testing.T) { EnableLoopbackDSR: true, }, }, - nwInfo: &network.NetworkInfo{}, + subnetInfos: []network.SubnetInfo{}, ipconfigs: []*network.IPConfig{ { Address: func() net.IPNet { @@ -309,8 +309,8 @@ func TestDSRPolciy(t *testing.T) { { name: "test disable dsr policy", args: PolicyArgs{ - nwCfg: &cni.NetworkConfig{}, - nwInfo: &network.NetworkInfo{}, + nwCfg: &cni.NetworkConfig{}, + subnetInfos: []network.SubnetInfo{}, ipconfigs: []*network.IPConfig{ { Address: func() net.IPNet { @@ -341,7 +341,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { plugin *NetPlugin netNs string nwCfg *cni.NetworkConfig - ipamAddResult *IPAMAddResult + interfaceInfo *network.InterfaceInfo want string wantErr bool }{ @@ -360,24 +360,20 @@ func TestGetNetworkNameFromCNS(t *testing.T) { Name: "azure", MultiTenancy: true, }, - ipamAddResult: &IPAMAddResult{ - interfaceInfo: []network.InterfaceInfo{ + interfaceInfo: &network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ { - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.240.0.5"), - Mask: net.CIDRMask(24, 32), - }, - }, - }, - NCResponse: &cns.GetNetworkContainerResponse{ - MultiTenancyInfo: cns.MultiTenancyInfo{ - ID: 1, - }, + Address: net.IPNet{ + IP: net.ParseIP("10.240.0.5"), + Mask: net.CIDRMask(24, 32), }, }, }, + NCResponse: &cns.GetNetworkContainerResponse{ + MultiTenancyInfo: cns.MultiTenancyInfo{ + ID: 1, + }, + }, }, want: "azure-vlan1-10-240-0-0_24", wantErr: false, @@ -397,24 +393,20 @@ func TestGetNetworkNameFromCNS(t *testing.T) { Name: "azure", MultiTenancy: true, }, - ipamAddResult: &IPAMAddResult{ - interfaceInfo: []network.InterfaceInfo{ + interfaceInfo: &network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ { - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP(""), - Mask: net.CIDRMask(24, 32), - }, - }, - }, - NCResponse: &cns.GetNetworkContainerResponse{ - MultiTenancyInfo: cns.MultiTenancyInfo{ - ID: 1, - }, + Address: net.IPNet{ + IP: net.ParseIP(""), + Mask: net.CIDRMask(24, 32), }, }, }, + NCResponse: &cns.GetNetworkContainerResponse{ + MultiTenancyInfo: cns.MultiTenancyInfo{ + ID: 1, + }, + }, }, want: "", wantErr: true, @@ -434,24 +426,20 @@ func TestGetNetworkNameFromCNS(t *testing.T) { Name: "azure", MultiTenancy: true, }, - ipamAddResult: &IPAMAddResult{ - interfaceInfo: []network.InterfaceInfo{ + interfaceInfo: &network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ { - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.0.00.6"), - Mask: net.CIDRMask(24, 32), - }, - }, - }, - NCResponse: &cns.GetNetworkContainerResponse{ - MultiTenancyInfo: cns.MultiTenancyInfo{ - ID: 1, - }, + Address: net.IPNet{ + IP: net.ParseIP("10.0.00.6"), + Mask: net.CIDRMask(24, 32), }, }, }, + NCResponse: &cns.GetNetworkContainerResponse{ + MultiTenancyInfo: cns.MultiTenancyInfo{ + ID: 1, + }, + }, }, want: "", wantErr: true, @@ -471,24 +459,20 @@ func TestGetNetworkNameFromCNS(t *testing.T) { Name: "azure", MultiTenancy: true, }, - ipamAddResult: &IPAMAddResult{ - interfaceInfo: []network.InterfaceInfo{ + interfaceInfo: &network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ { - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.0.0.6"), - Mask: net.CIDRMask(24, 32), - }, - }, - }, - NCResponse: &cns.GetNetworkContainerResponse{ - MultiTenancyInfo: cns.MultiTenancyInfo{ - ID: 1, - }, + Address: net.IPNet{ + IP: net.ParseIP("10.0.0.6"), + Mask: net.CIDRMask(24, 32), }, }, }, + NCResponse: &cns.GetNetworkContainerResponse{ + MultiTenancyInfo: cns.MultiTenancyInfo{ + ID: 1, + }, + }, }, want: "", wantErr: true, @@ -508,20 +492,16 @@ func TestGetNetworkNameFromCNS(t *testing.T) { Name: "azure", MultiTenancy: false, }, - ipamAddResult: &IPAMAddResult{ - interfaceInfo: []network.InterfaceInfo{ + interfaceInfo: &network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ { - IPConfigs: []*network.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.0.0.6"), - Mask: net.CIDRMask(24, 32), - }, - }, + Address: net.IPNet{ + IP: net.ParseIP("10.0.0.6"), + Mask: net.CIDRMask(24, 32), }, - NCResponse: &cns.GetNetworkContainerResponse{}, }, }, + NCResponse: &cns.GetNetworkContainerResponse{}, }, want: "azure", wantErr: false, @@ -531,7 +511,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - networkName, err := tt.plugin.getNetworkName(tt.netNs, tt.ipamAddResult, tt.nwCfg) + networkName, err := tt.plugin.getNetworkName(tt.netNs, tt.interfaceInfo, tt.nwCfg) if tt.wantErr { require.Error(t, err) } else { diff --git a/network/endpoint_test.go b/network/endpoint_test.go index 3de6cf58fd..fd87f71fa4 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -184,7 +184,7 @@ var _ = Describe("Test Endpoint", func() { It("Should be added", func() { // Add endpoint with valid id ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), epInfo) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.EndpointID)) @@ -196,7 +196,7 @@ var _ = Describe("Test Endpoint", func() { extIf: &externalInterface{IPv4Gateway: net.ParseIP("192.168.0.1")}, } ep, err := nw2.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), epInfo) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.EndpointID)) @@ -212,7 +212,7 @@ var _ = Describe("Test Endpoint", func() { Expect(err).ToNot(HaveOccurred()) // Adding endpoint with same id should fail and delete should cleanup the state ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), epInfo) Expect(err).To(HaveOccurred()) Expect(ep2).To(BeNil()) assert.Contains(GinkgoT(), err.Error(), "Endpoint already exists") @@ -222,7 +222,7 @@ var _ = Describe("Test Endpoint", func() { // Adding an endpoint with an id. mockCli := NewMockEndpointClient(nil) ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), epInfo) Expect(err).ToNot(HaveOccurred()) Expect(ep2).ToNot(BeNil()) Expect(len(mockCli.endpoints)).To(Equal(1)) @@ -253,11 +253,11 @@ var _ = Describe("Test Endpoint", func() { } return nil - }), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) + }), NewMockNamespaceClient(), iptables.NewClient(), epInfo) Expect(err).To(HaveOccurred()) Expect(ep).To(BeNil()) ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}, 0) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), epInfo) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.EndpointID)) diff --git a/network/network_test.go b/network/network_test.go index 37fdcd8605..9da8b20788 100644 --- a/network/network_test.go +++ b/network/network_test.go @@ -100,7 +100,7 @@ var _ = Describe("Test Network", func() { nm := &networkManager{ ExternalInterfaces: map[string]*externalInterface{}, } - nwInfo := &NetworkInfo{ + nwInfo := &EndpointInfo{ MasterIfName: "eth0", } _, _ = nm.newNetwork(nwInfo) @@ -113,7 +113,7 @@ var _ = Describe("Test Network", func() { nm := &networkManager{ ExternalInterfaces: map[string]*externalInterface{}, } - nwInfo := &NetworkInfo{ + nwInfo := &EndpointInfo{ MasterIfName: "eth0", } nw, err := nm.newNetwork(nwInfo) @@ -127,7 +127,7 @@ var _ = Describe("Test Network", func() { nm := &networkManager{ ExternalInterfaces: map[string]*externalInterface{}, } - nwInfo := &NetworkInfo{ + nwInfo := &EndpointInfo{ Subnets: []SubnetInfo{{ Prefix: net.IPNet{ IP: net.IPv4(10, 0, 0, 1), @@ -150,8 +150,8 @@ var _ = Describe("Test Network", func() { Networks: map[string]*network{}, } nm.ExternalInterfaces["eth0"].Networks["nw"] = &network{} - nwInfo := &NetworkInfo{ - Id: "nw", + nwInfo := &EndpointInfo{ + NetworkId: "nw", MasterIfName: "eth0", } nw, err := nm.newNetwork(nwInfo) @@ -169,8 +169,8 @@ var _ = Describe("Test Network", func() { nm.ExternalInterfaces["eth0"] = &externalInterface{ Networks: map[string]*network{}, } - nwInfo := &NetworkInfo{ - Id: "nw", + nwInfo := &EndpointInfo{ + NetworkId: "nw", MasterIfName: "eth0", Mode: opModeTransparent, IPV6Mode: IPV6Nat, @@ -178,7 +178,7 @@ var _ = Describe("Test Network", func() { nw, err := nm.newNetwork(nwInfo) Expect(err).To(BeNil()) Expect(nw).NotTo(BeNil()) - Expect(nw.Id).To(Equal(nwInfo.Id)) + Expect(nw.Id).To(Equal(nwInfo.NetworkId)) }) }) @@ -191,8 +191,8 @@ var _ = Describe("Test Network", func() { nm.ExternalInterfaces["eth0"] = &externalInterface{ Networks: map[string]*network{}, } - nwInfo := &NetworkInfo{ - Id: "nw", + nwInfo := &EndpointInfo{ + NetworkId: "nw", MasterIfName: "eth0", Mode: opModeTransparentVlan, } diff --git a/network/network_windows_test.go b/network/network_windows_test.go index b52ef482a0..49a9488e43 100644 --- a/network/network_windows_test.go +++ b/network/network_windows_test.go @@ -34,8 +34,8 @@ func TestNewAndDeleteNetworkImplHnsV2(t *testing.T) { // we do this to avoid passing around os specific objects in platform agnostic code Hnsv2 = hnswrapper.NewHnsv2wrapperFake() - nwInfo := &NetworkInfo{ - Id: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + nwInfo := &EndpointInfo{ + NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -75,8 +75,8 @@ func TestSuccesfulNetworkCreationWhenAlreadyExists(t *testing.T) { _, err := Hnsv2.CreateNetwork(network) // network name is derived from network info id - nwInfo := &NetworkInfo{ - Id: "azure-vlan1-172-28-1-0_24", + nwInfo := &EndpointInfo{ + NetworkId: "azure-vlan1-172-28-1-0_24", MasterIfName: "eth0", Mode: "bridge", } @@ -108,8 +108,8 @@ func TestNewNetworkImplHnsV2WithTimeout(t *testing.T) { HnsCallTimeout: 10 * time.Second, } - nwInfo := &NetworkInfo{ - Id: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + nwInfo := &EndpointInfo{ + NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -131,8 +131,8 @@ func TestDeleteNetworkImplHnsV2WithTimeout(t *testing.T) { ExternalInterfaces: map[string]*externalInterface{}, } - nwInfo := &NetworkInfo{ - Id: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + nwInfo := &EndpointInfo{ + NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -180,8 +180,8 @@ func TestNewNetworkImplHnsV1WithTimeout(t *testing.T) { HnsCallTimeout: 5 * time.Second, } - nwInfo := &NetworkInfo{ - Id: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + nwInfo := &EndpointInfo{ + NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -203,8 +203,8 @@ func TestDeleteNetworkImplHnsV1WithTimeout(t *testing.T) { ExternalInterfaces: map[string]*externalInterface{}, } - nwInfo := &NetworkInfo{ - Id: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + nwInfo := &EndpointInfo{ + NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -260,8 +260,8 @@ func TestAddIPv6DefaultRoute(t *testing.T) { }, } - nwInfo := &NetworkInfo{ - Id: "d3f97a83-ba4c-45d5-ba88-dc56757ece28", + nwInfo := &EndpointInfo{ + NetworkId: "d3f97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", Subnets: networkSubnetInfo, @@ -303,8 +303,8 @@ func TestFailToAddIPv6DefaultRoute(t *testing.T) { }, } - nwInfo := &NetworkInfo{ - Id: "d3f97a83-ba4c-45d5-ba88-dc56757ece28", + nwInfo := &EndpointInfo{ + NetworkId: "d3f97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", Subnets: networkSubnetInfo, From b5b0a8f3bdd59c8ddcf28539a503324bff6145c1 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 2 Apr 2024 15:19:55 -0700 Subject: [PATCH 032/102] comment out endpoint test secondary endpoint client case to make tests compile --- network/endpoint_test.go | 75 ++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/network/endpoint_test.go b/network/endpoint_test.go index fd87f71fa4..8730b2ad9b 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -263,46 +263,47 @@ var _ = Describe("Test Endpoint", func() { Expect(ep.Id).To(Equal(epInfo.EndpointID)) }) }) - Context("When secondary endpoint client is used", func() { - _, ipnet, _ := net.ParseCIDR("0.0.0.0/0") - nw := &network{ - Endpoints: map[string]*endpoint{}, - Mode: opModeTransparent, - extIf: &externalInterface{BridgeName: "testbridge"}, - } - epInfo := &EndpointInfo{ - EndpointID: "768e8deb-eth1", - IfName: eth0IfName, - NICType: cns.InfraNIC, - } - secondaryEpInfo := &EndpointInfo{ - NICType: cns.DelegatedVMNIC, - Routes: []RouteInfo{{Dst: *ipnet}}, - } + // TODO: Add secondary endpoint client test coverage in a different way + // Context("When secondary endpoint client is used", func() { + // _, ipnet, _ := net.ParseCIDR("0.0.0.0/0") + // nw := &network{ + // Endpoints: map[string]*endpoint{}, + // Mode: opModeTransparent, + // extIf: &externalInterface{BridgeName: "testbridge"}, + // } + // epInfo := &EndpointInfo{ + // EndpointID: "768e8deb-eth1", + // IfName: eth0IfName, + // NICType: cns.InfraNIC, + // } + // secondaryEpInfo := &EndpointInfo{ + // NICType: cns.DelegatedVMNIC, + // Routes: []RouteInfo{{Dst: *ipnet}}, + // } - It("Should not endpoint to the network when there is an error", func() { - secondaryEpInfo.MacAddress = netio.BadHwAddr // mock netlink will fail to set link state on bad eth - ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal("SecondaryEndpointClient Error: " + netlink.ErrorMockNetlink.Error())) - Expect(ep).To(BeNil()) + // It("Should not endpoint to the network when there is an error", func() { + // secondaryEpInfo.MacAddress = netio.BadHwAddr // mock netlink will fail to set link state on bad eth + // ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), + // netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) + // Expect(err).To(HaveOccurred()) + // Expect(err.Error()).To(Equal("SecondaryEndpointClient Error: " + netlink.ErrorMockNetlink.Error())) + // Expect(ep).To(BeNil()) - secondaryEpInfo.MacAddress = netio.HwAddr - ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) - Expect(err).ToNot(HaveOccurred()) - Expect(ep.Id).To(Equal(epInfo.EndpointID)) - }) + // secondaryEpInfo.MacAddress = netio.HwAddr + // ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), + // netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) + // Expect(err).ToNot(HaveOccurred()) + // Expect(ep.Id).To(Equal(epInfo.EndpointID)) + // }) - It("Should add endpoint when there are no errors", func() { - secondaryEpInfo.MacAddress = netio.HwAddr - ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) - Expect(err).ToNot(HaveOccurred()) - Expect(ep.Id).To(Equal(epInfo.EndpointID)) - }) - }) + // It("Should add endpoint when there are no errors", func() { + // secondaryEpInfo.MacAddress = netio.HwAddr + // ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), + // netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) + // Expect(err).ToNot(HaveOccurred()) + // Expect(ep.Id).To(Equal(epInfo.EndpointID)) + // }) + // }) }) Describe("Test updateEndpoint", func() { From 59d9109c125afa97713c9dfe1f91f780165351b5 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 2 Apr 2024 17:06:18 -0700 Subject: [PATCH 033/102] address todos and comments from meeting --- cni/network/network.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index a8be26697d..3f6f261000 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -521,14 +521,9 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } - //TODO: remove - ifIndex, _ := findDefaultInterface(ipamAddResult) - if ifIndex == -1 { - ifIndex, _ = findDefaultInterface(ipamAddResult) - } - // This proably needs to be changed as we return all interfaces... - sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) + // TODO: This proably needs to be changed as we return all interfaces... + // sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) } ifIndex, _ := findDefaultInterface(ipamAddResult) @@ -573,9 +568,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return err } - if err != nil { - return err - } epInfos = append(epInfos, epInfo) // TODO: should this statement be based on the current iteration instead of the constant ifIndex? sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", @@ -675,19 +667,19 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn defaultInterfaceInfo := opt.ifInfo epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultInterfaceInfo.DNS, opt.k8sNamespace) // Probably won't panic if given bad values if err != nil { - // TODO: will it error out if we have a secondary endpoint that has blank DNS though? If so, problem! + // TODO: will it error out if we have a secondary endpoint that has blank DNS though? If so, problem! Discussed and answer is no. err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) return nil, err } policyArgs := PolicyArgs{ // pass podsubnet info etc. part of epinfo - subnetInfos: nwInfo.Subnets, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? getEndpointPolicies requires nwInfo.Subnets only (checked) + subnetInfos: nwInfo.Subnets, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? Discussed and answer is it's fine. getEndpointPolicies requires nwInfo.Subnets only (checked) nwCfg: opt.nwCfg, ipconfigs: defaultInterfaceInfo.IPConfigs, } endpointPolicies, err := getEndpointPolicies(policyArgs) if err != nil { - // TODO: should only error out if we have an ip config and it is not readable + // TODO: should only error out if we have an ip config and it is not readable. Discussed and answer is it's fine. logger.Error("Failed to get endpoint policies", zap.Error(err)) return nil, err } @@ -751,11 +743,13 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn if opt.ipamAddResult.ipv6Enabled { // not specific to this particular interface epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working } - + // TODO: REMOVE (any azIpam or azIpamResult.IPs should be removed, or infra vnet ip or epInfo.InfraVnetIP-- don't remove from endpoint or network state) + // PROBLEM: The infra vnet ip is used in OVS etc. (snat rules) if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address } + // TODO: Do I remove azIpamResult as a param from the function below? It is used in linux (changes ep info routes)! if opt.nwCfg.MultiTenancy { plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, defaultInterfaceInfo) } From 655ccee0ae9cdb6605ffb904ca7f371795111162 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 3 Apr 2024 10:02:54 -0700 Subject: [PATCH 034/102] remove duplicated code for populating address in ep info generation --- cni/network/network.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 3f6f261000..8458192776 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -694,7 +694,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn vethName = fmt.Sprintf("%s%s%s", nwInfo.NetworkId, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) We use the nwInfo we generated above } - // for secondary + // for secondary (Populate addresses) var addresses []net.IPNet for _, ipconfig := range opt.ifInfo.IPConfigs { addresses = append(addresses, ipconfig.Address) @@ -735,11 +735,6 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn } epInfo.Policies = append(epInfo.Policies, epPolicies...) - // Populate addresses. - for _, ipconfig := range defaultInterfaceInfo.IPConfigs { - epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) - } - if opt.ipamAddResult.ipv6Enabled { // not specific to this particular interface epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working } From fe071bda7b20b977867fccabe7847058d3d0f4fb Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 3 Apr 2024 15:15:31 -0700 Subject: [PATCH 035/102] update EndpointCreate to support multiple infra nic --- cni/network/network.go | 2 ++ network/manager.go | 30 +++++++++++++++++++----------- network/manager_mock.go | 2 +- network/network.go | 40 +++++++++++++++++++++------------------- 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 8458192776..8ae18ef275 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -766,6 +766,8 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn epInfo.IsIPv6Enabled = nwInfo.IsIPv6Enabled epInfo.NetworkDNS = nwInfo.NetworkDNS + logger.Info("Generated endpoint info from fields", zap.String("epInfo", epInfo.PrettyString())) + // now our ep info should have the full combined information from both the network and endpoint structs return &epInfo, nil } diff --git a/network/manager.go b/network/manager.go index 94b4637e5c..cb06be5b49 100644 --- a/network/manager.go +++ b/network/manager.go @@ -110,7 +110,7 @@ type NetworkManager interface { GetNumberOfEndpoints(ifName string, networkID string) int GetEndpointID(containerID, ifName string) string IsStatelessCNIMode() bool - SaveState(networkID string, ep *endpoint) error + SaveState(eps map[string]*endpoint) error } // Creates a new network manager. @@ -687,26 +687,34 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string { } return containerID + "-" + ifName } -func (nm *networkManager) SaveState(networkID string, ep *endpoint) error { + +// saves the map of network ids to endpoints to the state file +func (nm *networkManager) SaveState(eps map[string]*endpoint) error { // TODO: Necessary? nm.Lock() defer nm.Unlock() - nw, err := nm.getNetwork(networkID) - if err != nil { - return err - } + logger.Info("Saving state") + // TODO: What if we fail halfway through? + for networkID, ep := range eps { + nw, err := nm.getNetwork(networkID) + if err != nil { + return err + } - nw.Endpoints[ep.Id] = ep // used only once + nw.Endpoints[ep.Id] = ep // used only once - if nm.IsStatelessCNIMode() { - err = nm.UpdateEndpointState(ep) - return err + if nm.IsStatelessCNIMode() { + err = nm.UpdateEndpointState(ep) + return err + } } - err = nm.save() + // once endpoints and networks are in-memory, save once + err := nm.save() if err != nil { return err } + return nil } diff --git a/network/manager_mock.go b/network/manager_mock.go index 4654cc5929..5e5bd791fd 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -156,7 +156,7 @@ func (nm *MockNetworkManager) GetNumEndpointsByContainerID(_ string) int { } // TODO: understand mock behavior -func (nm *MockNetworkManager) SaveState(networkID string, ep *endpoint) error { +func (nm *MockNetworkManager) SaveState(eps map[string]*endpoint) error { return nil } diff --git a/network/network.go b/network/network.go index 9cbd4c4a29..fcd72a31d9 100644 --- a/network/network.go +++ b/network/network.go @@ -301,17 +301,19 @@ func (nm *networkManager) GetNumEndpointsByContainerID(containerID string) int { // networkID is opt.nwInfo.ID // cns url is opt.nwCfg.CNSUrl -// actually creates the network and corresponding endpoint +// Creates the network and corresponding endpoint func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*EndpointInfo) error { - interfaceInfos := []*InterfaceInfo{} - var networkIDofChosenEndpoint string - var epToSave *endpoint + interfaceInfos := map[string][]*InterfaceInfo{} // map of network ids to slices of interface info pointers (for secondary interfaces) + + epsToSave := map[string]*endpoint{} // mapping network id to endpoint object (for non-secondary/infra interfaces) + for _, epInfo := range epInfos { + logger.Info("Creating endpoint and network", zap.String("endpointInfo", epInfo.PrettyString())) // check if network exists nwInfo, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkId) if nwGetErr != nil { + logger.Info("Existing network not found", zap.String("networkID", epInfo.NetworkId)) // Create the network if it is not found - // populate network info with fields from ep info err := nm.CreateNetwork(epInfo) if err != nil { // TODO: error messages/handling are different in this file @@ -321,10 +323,8 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo } // Create the endpoint. - logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) // sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) - // TODO: pass in network id, or get network id from ep info-- it makes more sense if it came from epInfo - // since the network is directly related to the epInfo + // nwInfo.NetworkId is the same as epInfo.NetworkId ep, err := nm.CreateEndpoint(cnsclient, nwInfo.NetworkId, epInfo) if err != nil { // err = plugin.Errorf("Failed to create endpoint: %v", err) @@ -332,16 +332,17 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo } // if infra nic, we will use this endpoint for its root fields, and otherwise, we collect all secondary/delegated if infos to add to the ep secondary interface slice if epInfo.NICType == cns.InfraNIC { - epToSave = ep - networkIDofChosenEndpoint = epInfo.NetworkId + // if there are multiple endpoints to be created (in dual nic case), we assume they will be on different networks + epsToSave[epInfo.NetworkId] = ep } else { - // taken from secondary endpoint client + // we discard the endpoint object created + // below is taken from secondary endpoint client ipconfigs := make([]*IPConfig, len(epInfo.IPAddresses)) for i, ipconfig := range epInfo.IPAddresses { ipconfigs[i] = &IPConfig{Address: ipconfig} } - - interfaceInfos = append(interfaceInfos, &InterfaceInfo{ + // can append to nil slice + interfaceInfos[epInfo.NetworkId] = append(interfaceInfos[epInfo.NetworkId], &InterfaceInfo{ Name: epInfo.IfName, MacAddress: epInfo.MacAddress, IPConfigs: ipconfigs, @@ -351,12 +352,13 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo } } // convert to one endpoint - epToSave.SecondaryInterfaces = map[string]*InterfaceInfo{} - for _, ifInfo := range interfaceInfos { - epToSave.SecondaryInterfaces[ifInfo.Name] = ifInfo + for networkID, ep := range epsToSave { + ep.SecondaryInterfaces = map[string]*InterfaceInfo{} + for _, ifInfo := range interfaceInfos[networkID] { + ep.SecondaryInterfaces[ifInfo.Name] = ifInfo + } } - // save - nm.SaveState(networkIDofChosenEndpoint, epToSave) - return nil + // save endpoint (non dual nic case) or endpoints (dual nic case) + return nm.SaveState(epsToSave) } From 1f67a112d93cead14ea25dca8f5b0076d3fba9fc Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 3 Apr 2024 17:52:15 -0700 Subject: [PATCH 036/102] save all endpoints to state, regardless of type, use either stateless or cni statefile undos some changes in "move create endpoint to network package, remove ifIndex as needed" deletion flow needs to figure out how to tell if the nic type is delegated 1 interface info : 1 endpoint info : 1 endpoint struct mapping --- network/endpoint.go | 1 + network/manager.go | 38 ++++++++++++-------------------------- network/network.go | 38 +++++--------------------------------- 3 files changed, 18 insertions(+), 59 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index 077e5ef40e..d086deabe3 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -182,6 +182,7 @@ func (nw *network) newEndpoint( return nil, err } + nw.Endpoints[ep.Id] = ep logger.Info("Created endpoint. Num of endpoints", zap.Any("ep", ep), zap.Int("numEndpoints", len(nw.Endpoints))) return ep, nil } diff --git a/network/manager.go b/network/manager.go index cb06be5b49..c71d7af7f6 100644 --- a/network/manager.go +++ b/network/manager.go @@ -110,7 +110,7 @@ type NetworkManager interface { GetNumberOfEndpoints(ifName string, networkID string) int GetEndpointID(containerID, ifName string) string IsStatelessCNIMode() bool - SaveState(eps map[string]*endpoint) error + SaveState(eps []*endpoint) error } // Creates a new network manager. @@ -396,14 +396,6 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } }() - if nm.IsStatelessCNIMode() { - err = nm.UpdateEndpointState(ep) - if err != nil { - return nil, err - } - // TODO: If stateless cni success, we still return ep right - } - return ep, nil } @@ -689,32 +681,26 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string { } // saves the map of network ids to endpoints to the state file -func (nm *networkManager) SaveState(eps map[string]*endpoint) error { +func (nm *networkManager) SaveState(eps []*endpoint) error { // TODO: Necessary? nm.Lock() defer nm.Unlock() logger.Info("Saving state") // TODO: What if we fail halfway through? - for networkID, ep := range eps { - nw, err := nm.getNetwork(networkID) - if err != nil { - return err - } - - nw.Endpoints[ep.Id] = ep // used only once - + for _, ep := range eps { if nm.IsStatelessCNIMode() { - err = nm.UpdateEndpointState(ep) - return err + err := nm.UpdateEndpointState(ep) + if err != nil { + return err + } } } - - // once endpoints and networks are in-memory, save once - err := nm.save() - if err != nil { - return err + // we either use stateless cni and save via update endpoint state, or use the state file + if nm.IsStatelessCNIMode() { + return nil } - return nil + // once endpoints and networks are in-memory, save once + return nm.save() } diff --git a/network/network.go b/network/network.go index fcd72a31d9..35f3ec4068 100644 --- a/network/network.go +++ b/network/network.go @@ -8,7 +8,6 @@ import ( "net" "strings" - "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" "go.uber.org/zap" @@ -303,9 +302,7 @@ func (nm *networkManager) GetNumEndpointsByContainerID(containerID string) int { // cns url is opt.nwCfg.CNSUrl // Creates the network and corresponding endpoint func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*EndpointInfo) error { - interfaceInfos := map[string][]*InterfaceInfo{} // map of network ids to slices of interface info pointers (for secondary interfaces) - - epsToSave := map[string]*endpoint{} // mapping network id to endpoint object (for non-secondary/infra interfaces) + eps := []*endpoint{} // mapping network id to endpoint object (for non-secondary/infra interfaces) for _, epInfo := range epInfos { logger.Info("Creating endpoint and network", zap.String("endpointInfo", epInfo.PrettyString())) @@ -330,35 +327,10 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo // err = plugin.Errorf("Failed to create endpoint: %v", err) return err //added } - // if infra nic, we will use this endpoint for its root fields, and otherwise, we collect all secondary/delegated if infos to add to the ep secondary interface slice - if epInfo.NICType == cns.InfraNIC { - // if there are multiple endpoints to be created (in dual nic case), we assume they will be on different networks - epsToSave[epInfo.NetworkId] = ep - } else { - // we discard the endpoint object created - // below is taken from secondary endpoint client - ipconfigs := make([]*IPConfig, len(epInfo.IPAddresses)) - for i, ipconfig := range epInfo.IPAddresses { - ipconfigs[i] = &IPConfig{Address: ipconfig} - } - // can append to nil slice - interfaceInfos[epInfo.NetworkId] = append(interfaceInfos[epInfo.NetworkId], &InterfaceInfo{ - Name: epInfo.IfName, - MacAddress: epInfo.MacAddress, - IPConfigs: ipconfigs, - NICType: epInfo.NICType, - SkipDefaultRoutes: epInfo.SkipDefaultRoutes, - }) - } - } - // convert to one endpoint - for networkID, ep := range epsToSave { - ep.SecondaryInterfaces = map[string]*InterfaceInfo{} - for _, ifInfo := range interfaceInfos[networkID] { - ep.SecondaryInterfaces[ifInfo.Name] = ifInfo - } + + eps = append(eps, ep) } - // save endpoint (non dual nic case) or endpoints (dual nic case) - return nm.SaveState(epsToSave) + // save endpoints + return nm.SaveState(eps) } From 9945d1e92a9314ce0891979c2e5f602baa925d5a Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 4 Apr 2024 17:17:48 -0700 Subject: [PATCH 037/102] fix dual nic support conditional and finding master interface ip the master interface ip must be in a particular form where the last few bits are zeroed out based on the mask or we won't find the ip for example, while the host subnet perfix is 10.224.0.113/16, the ip that should be passed into find master interface (subnet) should be 10.224.0.0/16 which matches one of the interfaces' ipnet (10.224.0.0/16) --- cni/network/network.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 8ae18ef275..1d0e1dedd4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -498,8 +498,8 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { zap.Error(err)) return err } - - if !plugin.isDualNicFeatureSupported(args.Netns) { + // dual nic when we get multiple interface infos back (multitenancy does not necessarily have multiple if infos) + if len(ipamAddResult.interfaceInfo) > 1 && !plugin.isDualNicFeatureSupported(args.Netns) { errMsg := fmt.Sprintf("received multiple NC results %+v from CNS while dualnic feature is not supported", ipamAddResults) logger.Error("received multiple NC results from CNS while dualnic feature is not supported", zap.Any("results", ipamAddResult)) @@ -620,7 +620,9 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn epInfo network.EndpointInfo nwInfo network.EndpointInfo ) - + // ensure we can find the master interface + opt.ifInfo.HostSubnetPrefix.IP = opt.ifInfo.HostSubnetPrefix.IP.Mask(opt.ifInfo.HostSubnetPrefix.Mask) + opt.ipamAddConfig.nwCfg.IPAM.Subnet = opt.ifInfo.HostSubnetPrefix.String() // populate endpoint info section masterIfName := plugin.findMasterInterface(opt) if masterIfName == "" { From 608b372750d198a46b09d550699246e3ec361b40 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 4 Apr 2024 18:32:32 -0700 Subject: [PATCH 038/102] fix empty network name when we need to create a network, we collect the network information, but if we do not find the network, we return an empty nw info and an error when we create the endpoint we need to use endpoint info's network id, not the (possibly) empty network info struct's network id --- cni/network/network.go | 6 ++++-- network/network.go | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 1d0e1dedd4..c8d80a8f73 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -578,8 +578,10 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return errors.Wrap(err, "failed to create cns client") } - plugin.nm.EndpointCreate(cnsclient, epInfos) - + err = plugin.nm.EndpointCreate(cnsclient, epInfos) + if err != nil { + return err // behavior can change if you don't assign to err prior to returning + } return nil } diff --git a/network/network.go b/network/network.go index 35f3ec4068..e40c449df8 100644 --- a/network/network.go +++ b/network/network.go @@ -307,7 +307,7 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo for _, epInfo := range epInfos { logger.Info("Creating endpoint and network", zap.String("endpointInfo", epInfo.PrettyString())) // check if network exists - nwInfo, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkId) + _, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkId) if nwGetErr != nil { logger.Info("Existing network not found", zap.String("networkID", epInfo.NetworkId)) // Create the network if it is not found @@ -322,7 +322,7 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo // Create the endpoint. // sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) // nwInfo.NetworkId is the same as epInfo.NetworkId - ep, err := nm.CreateEndpoint(cnsclient, nwInfo.NetworkId, epInfo) + ep, err := nm.CreateEndpoint(cnsclient, epInfo.NetworkId, epInfo) if err != nil { // err = plugin.Errorf("Failed to create endpoint: %v", err) return err //added From 4686fe4e5ff900ec0b6c34ef7cb12c784760d7a0 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 5 Apr 2024 09:26:50 -0700 Subject: [PATCH 039/102] make network_test.go compile (linux and windows compile) unit tests are not necessarily correct at this point --- network/manager.go | 1 - network/manager_mock.go | 2 +- network/network.go | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/network/manager.go b/network/manager.go index c71d7af7f6..ad62eac0f9 100644 --- a/network/manager.go +++ b/network/manager.go @@ -682,7 +682,6 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string { // saves the map of network ids to endpoints to the state file func (nm *networkManager) SaveState(eps []*endpoint) error { - // TODO: Necessary? nm.Lock() defer nm.Unlock() diff --git a/network/manager_mock.go b/network/manager_mock.go index 5e5bd791fd..14ec91ff96 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -156,7 +156,7 @@ func (nm *MockNetworkManager) GetNumEndpointsByContainerID(_ string) int { } // TODO: understand mock behavior -func (nm *MockNetworkManager) SaveState(eps map[string]*endpoint) error { +func (nm *MockNetworkManager) SaveState(eps []*endpoint) error { return nil } diff --git a/network/network.go b/network/network.go index e40c449df8..3b126f9d46 100644 --- a/network/network.go +++ b/network/network.go @@ -302,7 +302,7 @@ func (nm *networkManager) GetNumEndpointsByContainerID(containerID string) int { // cns url is opt.nwCfg.CNSUrl // Creates the network and corresponding endpoint func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*EndpointInfo) error { - eps := []*endpoint{} // mapping network id to endpoint object (for non-secondary/infra interfaces) + eps := []*endpoint{} // save endpoints for stateless for _, epInfo := range epInfos { logger.Info("Creating endpoint and network", zap.String("endpointInfo", epInfo.PrettyString())) From daae8c0d1d3c0e005df86b047dc2635146bae796 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 5 Apr 2024 10:17:28 -0700 Subject: [PATCH 040/102] add NICType to endpoint struct and populate it important: when getting the endpoint state, the NIC Type field is not populated, leading to deletes not having a NIC Type; this should be changed so that getting the state populates that field including the nic type allows us to simplify the secondary endpoints delete flow (just check if the nic type is delegated instead of checking if the secondary interfaces map is populated) smoke tested: linux aks podsubnet (same vm, multi vm, internet, cni statefile consistent) linux standalone transparent vlan multitenancy (same vm, multi vm, internet, multi vnet, no connection between coke pepsi, cni statefile consistent) windows standalone bridge multitenancy single customer (same vm connections, internet, dns only, cni statefile consistent, 2 pods deleting and recreating) --- network/endpoint.go | 2 ++ network/endpoint_linux.go | 4 +++- network/manager.go | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/network/endpoint.go b/network/endpoint.go index d086deabe3..5ded56fcaa 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -55,6 +55,8 @@ type endpoint struct { NetNs string `json:",omitempty"` // SecondaryInterfaces is a map of interface name to InterfaceInfo SecondaryInterfaces map[string]*InterfaceInfo + // Store nic type since we no longer populate SecondaryInterfaces + NICType cns.NICType } // EndpointInfo contains read-only information about an endpoint. diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 826a9386d5..f05ffaeffb 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -118,6 +118,7 @@ func (nw *network) newEndpointImpl( PODNameSpace: defaultEpInfo.PODNameSpace, Routes: defaultEpInfo.Routes, SecondaryInterfaces: make(map[string]*InterfaceInfo), + NICType: defaultEpInfo.NICType, } if nw.extIf != nil { ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} @@ -277,7 +278,8 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. } else if nw.Mode != opModeTransparent { epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } else { - if len(ep.SecondaryInterfaces) > 0 { + // delete if secondary interfaces populated or endpoint of type delegated (new way) + if len(ep.SecondaryInterfaces) > 0 || ep.NICType == cns.DelegatedVMNIC { epClient = NewSecondaryEndpointClient(nl, nioc, plc, nsc, ep) epClient.DeleteEndpointRules(ep) //nolint:errcheck // ignore error diff --git a/network/manager.go b/network/manager.go index ad62eac0f9..31bea5f573 100644 --- a/network/manager.go +++ b/network/manager.go @@ -427,6 +427,7 @@ func (nm *networkManager) GetEndpointState(networkID, endpointID string) (*Endpo PODNameSpace: endpointResponse.EndpointInfo.PodNamespace, NetworkContainerID: endpointID, HNSEndpointID: endpointResponse.EndpointInfo.HnsEndpointID, + // TODO: Populate NIC Type } for _, ip := range endpointResponse.EndpointInfo.IfnameToIPMap { @@ -493,6 +494,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint EnableSnatOnHost: false, EnableMultitenancy: false, NetworkContainerID: epInfo.EndpointID, + NICType: epInfo.NICType, } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) return nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) From ded5be56067a74a87b8f6bda7d4697d2f0e1357e Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Thu, 28 Mar 2024 17:06:15 -0700 Subject: [PATCH 041/102] ci: InterfaceInfo Map --- cni/network/invoker.go | 13 ++++- cni/network/invoker_azure.go | 20 ++++--- cni/network/invoker_azure_test.go | 8 +-- cni/network/invoker_cns.go | 58 +++++++++---------- cni/network/invoker_cns_test.go | 40 +++++-------- cni/network/invoker_mock.go | 39 +++++++------ cni/network/multitenancy.go | 20 ++++--- cni/network/multitenancy_mock.go | 21 ++++--- cni/network/network.go | 94 ++++++++++++++++++------------- network/endpoint.go | 5 ++ network/endpoint_test.go | 80 ++++++++++++++------------ network/manager_mock.go | 25 +++++++- 12 files changed, 236 insertions(+), 187 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index ea76889213..43cc9a2f2f 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -26,6 +26,15 @@ type IPAMAddConfig struct { } type IPAMAddResult struct { - interfaceInfo []network.InterfaceInfo - ipv6Enabled bool + interfaceInfo map[string]network.InterfaceInfo + // ncResponse and host subnet prefix were moved into interface info + ipv6Enabled bool +} + +func (ipamAddResult IPAMAddResult) PrettyString() string { + pStr := "InterfaceInfo: " + for _, ifInfo := range ipamAddResult.interfaceInfo { + pStr += ifInfo.PrettyString() + } + return pStr } diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index 67332ff933..016bf6601b 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -47,7 +47,7 @@ func NewAzureIpamInvoker(plugin *NetPlugin, nwInfo *network.EndpointInfo) *Azure } func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, error) { - addResult := IPAMAddResult{} + addResult := IPAMAddResult{interfaceInfo: make(map[string]network.InterfaceInfo)} if addConfig.nwCfg == nil { return addResult, invoker.plugin.Errorf("nil nwCfg passed to CNI ADD, stack: %+v", string(debug.Stack())) @@ -72,8 +72,8 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er defer func() { if err != nil { - if len(addResult.interfaceInfo[0].IPConfigs) > 0 { - if er := invoker.Delete(&addResult.interfaceInfo[0].IPConfigs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { + if len(addResult.interfaceInfo) > 0 && len(addResult.interfaceInfo[string(cns.InfraNIC)].IPConfigs) > 0 { + if er := invoker.Delete(&addResult.interfaceInfo[string(cns.InfraNIC)].IPConfigs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { err = invoker.plugin.Errorf("Failed to clean up IP's during Delete with error %v, after Add failed with error %w", er, err) } } else { @@ -113,18 +113,20 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er routes[i] = network.RouteInfo{Dst: route.Dst, Gw: route.GW} } - addResult.interfaceInfo = append(addResult.interfaceInfo, network.InterfaceInfo{ + // TODO: changed how host subnet prefix populated (check) + var hostSubnetPrefix = net.IPNet{} + if len(result.IPs) > 0 { + hostSubnetPrefix = result.IPs[0].Address + } + addResult.interfaceInfo[string(cns.InfraNIC)] = network.InterfaceInfo{ IPConfigs: ipconfigs, Routes: routes, DNS: network.DNSInfo{ Suffix: result.DNS.Domain, Servers: result.DNS.Nameservers, }, - NICType: cns.InfraNIC, - }) - - if len(result.IPs) > 0 { - addResult.interfaceInfo[0].HostSubnetPrefix = result.IPs[0].Address + NICType: cns.InfraNIC, + HostSubnetPrefix: hostSubnetPrefix, } return addResult, err diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 72cff01cc2..fae03b1b44 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -240,15 +240,9 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { } for _, ifInfo := range ipamAddResult.interfaceInfo { - switch ifInfo.NICType { - case cns.DelegatedVMNIC, cns.BackendNIC: - fmt.Print(errInvalidNIC) - require.FailNow("No coverage for this NICType") - case cns.InfraNIC: + if ifInfo.NICType == cns.InfraNIC { fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ifInfo.IPConfigs) require.Exactly(tt.want, ifInfo.IPConfigs) - default: - require.FailNow("Unsupported NICType") } } }) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 81a463dc9c..c29b76bcef 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -141,9 +141,8 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } } - addResult := IPAMAddResult{} + addResult := IPAMAddResult{interfaceInfo: make(map[string]network.InterfaceInfo)} numInterfacesWithDefaultRoutes := 0 - ifIndex := -1 for i := 0; i < len(response.PodIPInfo); i++ { info := IPResultInfo{ podIPAddress: response.PodIPInfo[i].PodIPConfig.IPAddress, @@ -164,6 +163,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro zap.Any("podInfo", podInfo)) //nolint:exhaustive // ignore exhaustive types check + // Do we want to leverage this lint skip in other places of our code? switch info.nicType { case cns.DelegatedVMNIC: // only handling single v4 PodIPInfo for DelegatedVMNICs at the moment, will have to update once v6 gets added @@ -174,17 +174,11 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig); err != nil { return IPAMAddResult{}, err } - default: - // Assign default index - if ifIndex == -1 { - // add a default Interface - // would put within configureDefaultAddResult() but we have a dependency on numInterfacesWithDefaultRoutes - addResult.interfaceInfo = append(addResult.interfaceInfo, network.InterfaceInfo{NICType: cns.InfraNIC}) - ifIndex = len(addResult.interfaceInfo) - 1 - } + case cns.InfraNIC: // only count dualstack interface once - if addResult.interfaceInfo[ifIndex].IPConfigs == nil { - addResult.interfaceInfo[ifIndex].IPConfigs = make([]*network.IPConfig, 0) + _, exist := addResult.interfaceInfo[string(info.nicType)] + if !exist { + addResult.interfaceInfo[string(info.nicType)] = network.InterfaceInfo{} if !info.skipDefaultRoutes { numInterfacesWithDefaultRoutes++ } @@ -194,6 +188,8 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro if err := configureDefaultAddResult(&info, &addConfig, &addResult, overlayMode); err != nil { return IPAMAddResult{}, err } + default: + // TODO Error check or continue? } } @@ -391,12 +387,18 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } } - ifIndex, err := findDefaultInterface(*addResult) if err != nil { logger.Error("Error finding InfraNIC interface", zap.Error(err)) return errors.Wrap(err, "error finding InfraNIC interface") } + + // get the name of the primary IP address + _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) + if err != nil { + return fmt.Errorf("unable to parse hostSubnet: %w", err) + } + if ip := net.ParseIP(info.podIPAddress); ip != nil { defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix if ip.To4() == nil { @@ -404,7 +406,8 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add addResult.ipv6Enabled = true } - addResult.interfaceInfo[ifIndex].IPConfigs = append(addResult.interfaceInfo[ifIndex].IPConfigs, + ipConfigs := addResult.interfaceInfo[string(info.nicType)].IPConfigs + ipConfigs = append(addResult.interfaceInfo[string(info.nicType)].IPConfigs, &network.IPConfig{ Address: net.IPNet{ IP: ip, @@ -418,27 +421,24 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add return getRoutesErr } + resRoute := addResult.interfaceInfo[string(info.nicType)].Routes if len(routes) > 0 { - addResult.interfaceInfo[ifIndex].Routes = append(addResult.interfaceInfo[ifIndex].Routes, routes...) + resRoute = append(addResult.interfaceInfo[string(info.nicType)].Routes, routes...) } else { // add default routes if none are provided - addResult.interfaceInfo[ifIndex].Routes = append(addResult.interfaceInfo[ifIndex].Routes, network.RouteInfo{ + resRoute = append(addResult.interfaceInfo[string(info.nicType)].Routes, network.RouteInfo{ Dst: defaultRouteDstPrefix, Gw: ncgw, }) } - - addResult.interfaceInfo[ifIndex].SkipDefaultRoutes = info.skipDefaultRoutes - } - - // get the name of the primary IP address - _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) - if err != nil { - return fmt.Errorf("unable to parse hostSubnet: %w", err) + addResult.interfaceInfo[string(info.nicType)] = network.InterfaceInfo{ + NICType: cns.InfraNIC, + SkipDefaultRoutes: info.skipDefaultRoutes, + IPConfigs: ipConfigs, + Routes: resRoute, + HostSubnetPrefix: *hostIPNet, + } } - addResult.interfaceInfo[ifIndex].HostSubnetPrefix = *hostIPNet - addResult.interfaceInfo[ifIndex].NICType = cns.InfraNIC // This can be removed - // set subnet prefix for host vm // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode // TODO: Remove v4overlay and dualstackoverlay options, after 'overlay' rolls out in AKS-RP @@ -467,7 +467,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p return err } - result := network.InterfaceInfo{ + addResult.interfaceInfo[string(info.macAddress)] = network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ { Address: net.IPNet{ @@ -482,7 +482,5 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p SkipDefaultRoutes: info.skipDefaultRoutes, } - addResult.interfaceInfo = append(addResult.interfaceInfo, result) - return nil } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 3268b9bf85..ea11f1aa3e 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -107,6 +107,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrimaryIP: "10.224.0.5", Subnet: "10.224.0.0/16", }, + NICType: cns.InfraNIC, }, Response: cns.Response{ ReturnCode: 0, @@ -175,6 +176,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, + NICType: cns.InfraNIC, }, { PodIPConfig: cns.IPSubnet{ @@ -194,6 +196,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrimaryIP: "fe80::1234:5678:9abc", Subnet: "fd11:1234::/112", }, + NICType: cns.InfraNIC, }, }, Response: cns.Response{ @@ -487,20 +490,15 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { } for _, ifInfo := range ipamAddResult.interfaceInfo { - switch ifInfo.NICType { - case cns.DelegatedVMNIC: + if ifInfo.NICType == cns.DelegatedVMNIC { fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ifInfo) if len(tt.wantSecondaryInterfacesInfo.IPConfigs) > 0 { require.EqualValues(tt.wantSecondaryInterfacesInfo, ifInfo, "incorrect multitenant response") } - case cns.BackendNIC: - fmt.Print(errInvalidNIC) - require.FailNow("No coverage for this NICType") - case cns.InfraNIC: - t.Logf("%s", &ifInfo.HostSubnetPrefix) + } + if ifInfo.NICType == cns.InfraNIC { + t.Logf("%s", &ifInfo.HostSubnetPrefix) // TODO: Remove later require.Equalf(tt.wantDefaultResult, ifInfo, "incorrect default response") - default: - require.FailNow("Unsupported NICType") } } }) @@ -559,6 +557,7 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, + NICType: cns.InfraNIC, }, }, Response: cns.Response{ @@ -626,6 +625,7 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, + NICType: cns.InfraNIC, }, { PodIPConfig: cns.IPSubnet{ @@ -645,6 +645,7 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { PrimaryIP: "fe80::1234:5678:9abc", Subnet: "fd11:1234::/112", }, + NICType: cns.InfraNIC, }, }, Response: cns.Response{ @@ -727,19 +728,14 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { } for _, ifInfo := range ipamAddResult.interfaceInfo { - switch ifInfo.NICType { - case cns.DelegatedVMNIC: + if ifInfo.NICType == cns.DelegatedVMNIC { fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ifInfo) if len(tt.wantMultitenantResult.IPConfigs) > 0 { require.Equalf(tt.wantMultitenantResult, ifInfo, "incorrect multitenant response") } - case cns.BackendNIC: - fmt.Print(errInvalidNIC) - require.FailNow("No coverage for this NICType") - case cns.InfraNIC: + } + if ifInfo.NICType == cns.InfraNIC { require.Equalf(tt.wantDefaultResult, ifInfo, "incorrect default response") - default: - require.FailNow("Unsupported NICType") } } }) @@ -771,7 +767,6 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { fields fields args args want network.InterfaceInfo - want1 network.InterfaceInfo // We dont use this anywhere..? Dead Code potentially, ask jaeryn wantErr bool }{ { @@ -803,6 +798,7 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, + NICType: cns.InfraNIC, }, Response: cns.Response{ ReturnCode: 0, @@ -859,14 +855,8 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { require.NoError(err) for _, ifInfo := range ipamAddResult.interfaceInfo { - switch ifInfo.NICType { - case cns.DelegatedVMNIC, cns.BackendNIC: - fmt.Print(errInvalidNIC) - require.FailNow("No coverage for this NICType") - case cns.InfraNIC: + if ifInfo.NICType == cns.InfraNIC { require.Equalf(tt.want, ifInfo, "incorrect ipv4 response") - default: - require.FailNow("Unsupported NICType") } } }) diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index ee303d8eff..a89e0463a6 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -21,8 +21,6 @@ var ( errV4 = errors.New("v4 fail") errV6 = errors.New("v6 Fail") errDelegatedVMNIC = errors.New("delegatedVMNIC fail") - errNoInfraNIC = errors.New("no InfraNIC fail") - errInvalidNIC = errors.New("no test case for this NIC") errDeleteIpam = errors.New("delete fail") ) @@ -51,32 +49,30 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes if invoker.v4Fail { return ipamAddResult, errV4 } + ipamAddResult = IPAMAddResult{interfaceInfo: make(map[string]network.InterfaceInfo)} + // TODO: the ipam add result host subnet prefix is in the interface info and ensure that when creating interface info, the host subnet prefix is set to an empty value (may break uts) ipv4Str := "10.240.0.5" if _, ok := invoker.ipMap["10.240.0.5/24"]; ok { ipv4Str = "10.240.0.6" } - ip := net.ParseIP(ipv4Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetBits, ipv4Bits)} gwIP := net.ParseIP("10.240.0.1") - ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{ - IPConfigs: []*network.IPConfig{ - {Address: ipnet, Gateway: gwIP}, - }, + ipRes := []*network.IPConfig{ + {Address: ipnet, Gateway: gwIP}, + } + + ipamAddResult.interfaceInfo[string(cns.InfraNIC)] = network.InterfaceInfo{ + IPConfigs: ipRes, NICType: cns.InfraNIC, HostSubnetPrefix: net.IPNet{}, - }) + } invoker.ipMap[ipnet.String()] = true if invoker.v6Fail { return ipamAddResult, errV6 } - ifIndex, err := findDefaultInterface(ipamAddResult) - if err != nil { - return IPAMAddResult{}, errNoInfraNIC - } - if invoker.isIPv6 { ipv6Str := "fc00::2" if _, ok := invoker.ipMap["fc00::2/128"]; ok { @@ -86,7 +82,11 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv6Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetv6Bits, ipv6Bits)} gwIP := net.ParseIP("fc00::1") - ipamAddResult.interfaceInfo[ifIndex].IPConfigs = append(ipamAddResult.interfaceInfo[ifIndex].IPConfigs, &network.IPConfig{Address: ipnet, Gateway: gwIP}) + ipRes = append(ipRes, &network.IPConfig{Address: ipnet, Gateway: gwIP}) + ipamAddResult.interfaceInfo[string(cns.InfraNIC)] = network.InterfaceInfo{ + IPConfigs: ipRes, + NICType: cns.InfraNIC, + } invoker.ipMap[ipnet.String()] = true } @@ -97,12 +97,11 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ipStr := "20.20.20.20/32" _, ipnet, _ := net.ParseCIDR(ipStr) - ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{ - IPConfigs: []*network.IPConfig{ - {Address: *ipnet}, - }, - NICType: cns.DelegatedVMNIC, - }) + ipRes = append(ipRes, &network.IPConfig{Address: *ipnet}) + ipamAddResult.interfaceInfo[string(cns.InfraNIC)] = network.InterfaceInfo{ + IPConfigs: ipRes, + NICType: cns.DelegatedVMNIC, + } } return ipamAddResult, nil diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 27d3805f25..8db247af91 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -216,20 +216,22 @@ func (m *Multitenancy) GetAllNetworkContainers( } ipamResult := IPAMAddResult{} - ipamResult.interfaceInfo = []network.InterfaceInfo{} + ipamResult.interfaceInfo = make(map[string]network.InterfaceInfo) - // ipamResults := make([]IPAMAddResult, len(ncResponses)) - // Can use hard coded 0 as ipamResults is empty and we are creating the first interface for i := 0; i < len(ncResponses); i++ { - ipamResult.interfaceInfo = append(ipamResult.interfaceInfo, network.InterfaceInfo{ + // one ncResponse gets you one interface info in the returned IPAMAddResult + ifInfo := network.InterfaceInfo{ NCResponse: &ncResponses[i], HostSubnetPrefix: hostSubnetPrefixes[i], - }) + } + + ipconfig, routes := convertToIPConfigAndRouteInfo(ifInfo.NCResponse) + ifInfo.IPConfigs = []*network.IPConfig{ipconfig} + ifInfo.Routes = routes + ifInfo.NICType = cns.InfraNIC - ipconfig, routes := convertToIPConfigAndRouteInfo(ipamResult.interfaceInfo[i].NCResponse) - ipamResult.interfaceInfo[i].IPConfigs = []*network.IPConfig{ipconfig} - ipamResult.interfaceInfo[i].Routes = routes - ipamResult.interfaceInfo[i].NICType = cns.InfraNIC + // assuming we only assign infra nics in this function + ipamResult.interfaceInfo[string(ifInfo.NICType)+string(i)] = ifInfo } return ipamResult, err diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index db5bf50550..b2202ce584 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -155,17 +155,22 @@ func (m *MockMultitenancy) GetAllNetworkContainers( cnsResponses = append(cnsResponses, *cnsResponseOne) ipamResult := IPAMAddResult{} - // Can use hard coded 0 as ipamResults is empty and we are creating the first interface + ipamResult.interfaceInfo = make(map[string]network.InterfaceInfo) + for i := 0; i < len(cnsResponses); i++ { - ipamResult.interfaceInfo = append(ipamResult.interfaceInfo, network.InterfaceInfo{ - NCResponse: &cnsResponses[i], - }) + // one ncResponse gets you one interface info in the returned IPAMAddResult + ifInfo := network.InterfaceInfo{ + NCResponse: &cnsResponses[i], + HostSubnetPrefix: ipNets[i], + } - ipconfig, routes := convertToIPConfigAndRouteInfo(ipamResult.interfaceInfo[i].NCResponse) - ipamResult.interfaceInfo[i].IPConfigs = []*network.IPConfig{ipconfig} - ipamResult.interfaceInfo[i].Routes = routes - ipamResult.interfaceInfo[i].NICType = cns.InfraNIC + ipconfig, routes := convertToIPConfigAndRouteInfo(ifInfo.NCResponse) + ifInfo.IPConfigs = []*network.IPConfig{ipconfig} + ifInfo.Routes = routes + ifInfo.NICType = cns.InfraNIC + // assuming we only assign infra nics in this function + ipamResult.interfaceInfo[string(ifInfo.NICType)+string(i)] = ifInfo } return ipamResult, nil diff --git a/cni/network/network.go b/cni/network/network.go index c8d80a8f73..b98717f23f 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -331,7 +331,7 @@ func (plugin *NetPlugin) addIpamInvoker(ipamAddConfig IPAMAddConfig) (IPAMAddRes if err != nil { return IPAMAddResult{}, err } - sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam interfaces: %+v", ipamAddResult.interfaceInfo)) + sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam interface: %+v", ipamAddResult.PrettyString())) return ipamAddResult, nil } @@ -401,28 +401,36 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { telemetry.SendCNIMetric(&cniMetric, plugin.tb) // Add Interfaces to result. - for _, intfInfo := range ipamAddResult.interfaceInfo { - cniResult := convertInterfaceInfoToCniResult(intfInfo, args.IfName) - addSnatInterface(nwCfg, cniResult) - // Convert result to the requested CNI version. - res, vererr := cniResult.GetAsVersion(nwCfg.CNIVersion) - if vererr != nil { - logger.Error("GetAsVersion failed", zap.Error(vererr)) - plugin.Error(vererr) + // previously just logged the default (infra) interface so this is equivalent behavior + cniResult := &cniTypesCurr.Result{} + for _, ifInfo := range ipamAddResult.interfaceInfo { + if ifInfo.NICType == cns.InfraNIC { + cniResult = convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[string(cns.InfraNIC)], args.IfName) } + } - if err == nil && res != nil { - // Output the result to stdout. - res.Print() - } + addSnatInterface(nwCfg, cniResult) - logger.Info("ADD command completed for", - zap.String("pod", k8sPodName), - zap.Any("IPs", cniResult.IPs), - zap.Error(err)) + // Convert result to the requested CNI version. + res, vererr := cniResult.GetAsVersion(nwCfg.CNIVersion) + if vererr != nil { + logger.Error("GetAsVersion failed", zap.Error(vererr)) + plugin.Error(vererr) + } + + if err == nil && res != nil { + // Output the result to stdout. + res.Print() } + + logger.Info("ADD command completed for", + zap.String("pod", k8sPodName), + zap.Any("IPs", cniResult.IPs), + zap.Error(err)) }() + ipamAddResult = IPAMAddResult{interfaceInfo: make(map[string]network.InterfaceInfo)} + // Parse Pod arguments. k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args) if err != nil { @@ -452,12 +460,11 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { res, err = plugin.nnsClient.AddContainerNetworking(context.Background(), k8sPodName, args.Netns) if err == nil { - ipamAddResult.interfaceInfo = append(ipamAddResult.interfaceInfo, network.InterfaceInfo{ + ipamAddResult.interfaceInfo[string(cns.InfraNIC)] = network.InterfaceInfo{ IPConfigs: convertNnsToIPConfigs(res, args.IfName, k8sPodName, "AddContainerNetworking"), NICType: cns.InfraNIC, - }) + } } - return err } @@ -526,15 +533,22 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) } - ifIndex, _ := findDefaultInterface(ipamAddResult) endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) + // moved to addIpamInvoker + // sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam interface: %+v", ipamAddResult.PrettyString())) defer func() { //nolint:gocritic if err != nil { // for multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips if !(nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS) { - plugin.cleanupAllocationOnError(ipamAddResult.interfaceInfo[ifIndex].IPConfigs, nwCfg, args, options) + for _, ifInfo := range ipamAddResult.interfaceInfo { + // This used to only be called for infraNIC, test if this breaks scenarios + // If it does then will have to search for infraNIC + if ifInfo.NICType == cns.InfraNIC { + plugin.cleanupAllocationOnError(ifInfo.IPConfigs, nwCfg, args, options) + } + } } } }() @@ -570,8 +584,8 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } epInfos = append(epInfos, epInfo) // TODO: should this statement be based on the current iteration instead of the constant ifIndex? - sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", - ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + // TODO figure out where to put telemetry: sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", + // ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) } cnsclient, err := cnscli.New(nwCfg.CNSUrl, defaultRequestTimeout) @@ -582,6 +596,8 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return err // behavior can change if you don't assign to err prior to returning } + // telemetry added + sendEvent(plugin, fmt.Sprintf("CNI ADD Process succeeded for interfaces: %v", ipamAddResult.PrettyString())) return nil } @@ -623,6 +639,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn nwInfo network.EndpointInfo ) // ensure we can find the master interface + // TODO: in swiftv2 with secondary interface, there won't be a host subnet prefix-- ensure this does not crash/break opt.ifInfo.HostSubnetPrefix.IP = opt.ifInfo.HostSubnetPrefix.IP.Mask(opt.ifInfo.HostSubnetPrefix.Mask) opt.ipamAddConfig.nwCfg.IPAM.Subnet = opt.ifInfo.HostSubnetPrefix.String() // populate endpoint info section @@ -668,8 +685,8 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // populate endpoint info // populate endpoint info fields code is below - defaultInterfaceInfo := opt.ifInfo - epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultInterfaceInfo.DNS, opt.k8sNamespace) // Probably won't panic if given bad values + //defaultInterfaceInfo := opt.ifInfo + epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, opt.ifInfo.DNS, opt.k8sNamespace) // Probably won't panic if given bad values if err != nil { // TODO: will it error out if we have a secondary endpoint that has blank DNS though? If so, problem! Discussed and answer is no. err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) @@ -679,7 +696,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // pass podsubnet info etc. part of epinfo subnetInfos: nwInfo.Subnets, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? Discussed and answer is it's fine. getEndpointPolicies requires nwInfo.Subnets only (checked) nwCfg: opt.nwCfg, - ipconfigs: defaultInterfaceInfo.IPConfigs, + ipconfigs: opt.ifInfo.IPConfigs, } endpointPolicies, err := getEndpointPolicies(policyArgs) if err != nil { @@ -699,6 +716,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn } // for secondary (Populate addresses) + // intially only for infra nic but now applied to all nic types var addresses []net.IPNet for _, ipconfig := range opt.ifInfo.IPConfigs { addresses = append(addresses, ipconfig.Address) @@ -726,7 +744,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn NATInfo: opt.natInfo, NICType: opt.ifInfo.NICType, SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, - Routes: defaultInterfaceInfo.Routes, + Routes: opt.ifInfo.Routes, // added the following for delegated vm nic IPAddresses: addresses, MacAddress: opt.ifInfo.MacAddress, @@ -750,7 +768,8 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // TODO: Do I remove azIpamResult as a param from the function below? It is used in linux (changes ep info routes)! if opt.nwCfg.MultiTenancy { - plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, defaultInterfaceInfo) + // previously only infra nic was passed into this function but now all nics are passed in (possibly breaks swift v2) + plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, opt.ifInfo) } setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) @@ -1419,12 +1438,11 @@ func convertCniResultToInterfaceInfo(result *cniTypesCurr.Result) network.Interf return interfaceInfo } -func findDefaultInterface(ipamAddResult IPAMAddResult) (int, error) { - for i := 0; i < len(ipamAddResult.interfaceInfo); i++ { - if ipamAddResult.interfaceInfo[i].NICType == cns.InfraNIC { - return i, nil - } - } - - return -1, errors.New("no NIC was of type InfraNIC") -} +// func findInfraNicInterfaceKey(ifInfos map[string]network.InterfaceInfo) network.InterfaceInfo { +// for _, ifInfo := range ifInfos { +// if ifInfo.NICType == cns.InfraNIC { +// return &ifInfo +// } +// } +// return nil +// } diff --git a/network/endpoint.go b/network/endpoint.go index 5ded56fcaa..6cb5228580 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -158,6 +158,11 @@ func (epInfo *EndpointInfo) PrettyString() string { epInfo.Gateways, epInfo.Data) } +func (ifInfo *InterfaceInfo) PrettyString() string { + return fmt.Sprintf("Name:%s NICType:%v MacAddr:%s IPConfigs:%+v Routes:%+v DNSInfo:%+v", + ifInfo.Name, ifInfo.NICType, ifInfo.MacAddress.String(), ifInfo.IPConfigs, ifInfo.Routes, ifInfo.DNS) +} + // NewEndpoint creates a new endpoint in the network. func (nw *network) newEndpoint( apipaCli apipaClient, diff --git a/network/endpoint_test.go b/network/endpoint_test.go index 8730b2ad9b..5e75bb8d5e 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -264,46 +264,52 @@ var _ = Describe("Test Endpoint", func() { }) }) // TODO: Add secondary endpoint client test coverage in a different way - // Context("When secondary endpoint client is used", func() { - // _, ipnet, _ := net.ParseCIDR("0.0.0.0/0") - // nw := &network{ - // Endpoints: map[string]*endpoint{}, - // Mode: opModeTransparent, - // extIf: &externalInterface{BridgeName: "testbridge"}, - // } - // epInfo := &EndpointInfo{ - // EndpointID: "768e8deb-eth1", - // IfName: eth0IfName, - // NICType: cns.InfraNIC, - // } - // secondaryEpInfo := &EndpointInfo{ - // NICType: cns.DelegatedVMNIC, - // Routes: []RouteInfo{{Dst: *ipnet}}, - // } + Context("When secondary endpoint client is used", func() { + _, ipnet, _ := net.ParseCIDR("0.0.0.0/0") + nw := &network{ + Endpoints: map[string]*endpoint{}, + Mode: opModeTransparent, + extIf: &externalInterface{BridgeName: "testbridge"}, + } + epInfo := &EndpointInfo{ + EndpointID: "768e8deb-eth1", + IfName: eth0IfName, + NICType: cns.InfraNIC, + } + secondaryEpInfo := &EndpointInfo{ + // TODO: should this have an endpoint info always in swift v2? It (even delegated/secondary infos!) must have an ep id always or it will panic + NICType: cns.DelegatedVMNIC, + Routes: []RouteInfo{{Dst: *ipnet}}, + } + + It("Should not endpoint to the network when there is an error", func() { + secondaryEpInfo.MacAddress = netio.BadHwAddr // mock netlink will fail to set link state on bad eth + ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), secondaryEpInfo) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("SecondaryEndpointClient Error: " + netlink.ErrorMockNetlink.Error())) + Expect(ep).To(BeNil()) - // It("Should not endpoint to the network when there is an error", func() { - // secondaryEpInfo.MacAddress = netio.BadHwAddr // mock netlink will fail to set link state on bad eth - // ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - // netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) - // Expect(err).To(HaveOccurred()) - // Expect(err.Error()).To(Equal("SecondaryEndpointClient Error: " + netlink.ErrorMockNetlink.Error())) - // Expect(ep).To(BeNil()) + secondaryEpInfo.MacAddress = netio.HwAddr + ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), secondaryEpInfo) + Expect(err).ToNot(HaveOccurred()) + Expect(ep.Id).To(Equal(epInfo.EndpointID)) + }) - // secondaryEpInfo.MacAddress = netio.HwAddr - // ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - // netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) - // Expect(err).ToNot(HaveOccurred()) - // Expect(ep.Id).To(Equal(epInfo.EndpointID)) - // }) + It("Should add endpoint when there are no errors", func() { + secondaryEpInfo.MacAddress = netio.HwAddr + ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), secondaryEpInfo) + Expect(err).ToNot(HaveOccurred()) + Expect(ep.Id).To(Equal(epInfo.EndpointID)) - // It("Should add endpoint when there are no errors", func() { - // secondaryEpInfo.MacAddress = netio.HwAddr - // ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - // netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}, 0) - // Expect(err).ToNot(HaveOccurred()) - // Expect(ep.Id).To(Equal(epInfo.EndpointID)) - // }) - // }) + ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), epInfo) + Expect(err).ToNot(HaveOccurred()) + Expect(ep.Id).To(Equal(epInfo.EndpointID)) + }) + }) }) Describe("Test updateEndpoint", func() { diff --git a/network/manager_mock.go b/network/manager_mock.go index 14ec91ff96..8cc1ab3c3a 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -157,9 +157,30 @@ func (nm *MockNetworkManager) GetNumEndpointsByContainerID(_ string) int { // TODO: understand mock behavior func (nm *MockNetworkManager) SaveState(eps []*endpoint) error { + // TODO: Mock behavior for saving the state separate from TestEndpointInfo/NetworkInfo map maybe return nil } -func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfo []*EndpointInfo) error { - return nil +// TODO: understand mock behavior +func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*EndpointInfo) error { + eps := []*endpoint{} + for _, epInfo := range epInfos { + _, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkId) + if nwGetErr != nil { + err := nm.CreateNetwork(epInfo) + if err != nil { + return err + } + } + + ep, err := nm.CreateEndpoint(client, epInfo.NetworkId, epInfo) + if err != nil { + // err = plugin.Errorf("Failed to create endpoint: %v", err) + return err //added + } + eps = append(eps, ep) + } + + // save endpoints + return nm.SaveState(eps) } From f5de076a844a04ed789e2d6022f06785c97869fa Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 8 Apr 2024 08:59:46 -0700 Subject: [PATCH 042/102] fix multitenancy_test ut by changing key --- cni/network/multitenancy.go | 2 +- cni/network/multitenancy_mock.go | 3 ++- cni/network/multitenancy_test.go | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 8db247af91..39b757bca3 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -231,7 +231,7 @@ func (m *Multitenancy) GetAllNetworkContainers( ifInfo.NICType = cns.InfraNIC // assuming we only assign infra nics in this function - ipamResult.interfaceInfo[string(ifInfo.NICType)+string(i)] = ifInfo + ipamResult.interfaceInfo[string(ifInfo.NICType)+fmt.Sprint(i)] = ifInfo } return ipamResult, err diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index b2202ce584..9a34d5a569 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -3,6 +3,7 @@ package network import ( "context" "errors" + "fmt" "net" "runtime" @@ -170,7 +171,7 @@ func (m *MockMultitenancy) GetAllNetworkContainers( ifInfo.NICType = cns.InfraNIC // assuming we only assign infra nics in this function - ipamResult.interfaceInfo[string(ifInfo.NICType)+string(i)] = ifInfo + ipamResult.interfaceInfo[string(ifInfo.NICType)+fmt.Sprint(i)] = ifInfo } return ipamResult, nil diff --git a/cni/network/multitenancy_test.go b/cni/network/multitenancy_test.go index 841bfaec57..42d2cb2540 100644 --- a/cni/network/multitenancy_test.go +++ b/cni/network/multitenancy_test.go @@ -548,9 +548,9 @@ func TestGetMultiTenancyCNIResult(t *testing.T) { require.Error(err) } require.NoError(err) - require.Exactly(tt.want1, got.interfaceInfo[0].NCResponse) - require.Exactly(tt.want2, got.interfaceInfo[1].NCResponse) - require.Exactly(tt.want3, got.interfaceInfo[0].HostSubnetPrefix) + require.Exactly(tt.want1, got.interfaceInfo[string(cns.InfraNIC)+"0"].NCResponse) + require.Exactly(tt.want2, got.interfaceInfo[string(cns.InfraNIC)+"1"].NCResponse) + require.Exactly(tt.want3, got.interfaceInfo[string(cns.InfraNIC)+"0"].HostSubnetPrefix) // check multiple responses tt.want5 = append(tt.want5, *tt.want1, *tt.want2) @@ -691,7 +691,7 @@ func TestGetMultiTenancyCNIResultUnsupportedAPI(t *testing.T) { t.Fatalf("expected an error %+v but none received", err) } require.NoError(err) - require.Exactly(tt.want, got.interfaceInfo[0].NCResponse) + require.Exactly(tt.want, got.interfaceInfo[string(cns.InfraNIC)+"0"].NCResponse) }) } } From 60888e10dcf3afa4abd0762311b50afd62acaf5c Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 8 Apr 2024 09:37:52 -0700 Subject: [PATCH 043/102] add endpoint id to secondary ep info test since we populate the id in the actual flow --- network/endpoint_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/network/endpoint_test.go b/network/endpoint_test.go index 5e75bb8d5e..724094da27 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -277,19 +277,20 @@ var _ = Describe("Test Endpoint", func() { NICType: cns.InfraNIC, } secondaryEpInfo := &EndpointInfo{ - // TODO: should this have an endpoint info always in swift v2? It (even delegated/secondary infos!) must have an ep id always or it will panic - NICType: cns.DelegatedVMNIC, - Routes: []RouteInfo{{Dst: *ipnet}}, + // When we create the secondary endpoint infos while looping over the interface infos, we pass in the same endpoint id + EndpointID: "768e8deb-eth1", + NICType: cns.DelegatedVMNIC, + Routes: []RouteInfo{{Dst: *ipnet}}, } - It("Should not endpoint to the network when there is an error", func() { + It("Should not add endpoint to the network when there is an error", func() { secondaryEpInfo.MacAddress = netio.BadHwAddr // mock netlink will fail to set link state on bad eth ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), secondaryEpInfo) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("SecondaryEndpointClient Error: " + netlink.ErrorMockNetlink.Error())) Expect(ep).To(BeNil()) - + // should not panic or error when going through the unified endpoint impl flow with only the delegated nic type fields secondaryEpInfo.MacAddress = netio.HwAddr ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), secondaryEpInfo) From 2a010255e6c972d9c6c16c002cea8d1895fb6d44 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 8 Apr 2024 11:37:40 -0700 Subject: [PATCH 044/102] fix cni network_test linux and ensure secondary create ep info does not break in network_test we pass in sample delegated (secondary) data to Add which we then create endpoint info from even with most fields empty, in linux, the ep info is created without erroring --- cni/network/network.go | 16 +++++----------- network/manager_mock.go | 4 ++-- network/network.go | 4 ++-- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index b98717f23f..c3b714bf25 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -639,7 +639,6 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn nwInfo network.EndpointInfo ) // ensure we can find the master interface - // TODO: in swiftv2 with secondary interface, there won't be a host subnet prefix-- ensure this does not crash/break opt.ifInfo.HostSubnetPrefix.IP = opt.ifInfo.HostSubnetPrefix.IP.Mask(opt.ifInfo.HostSubnetPrefix.Mask) opt.ipamAddConfig.nwCfg.IPAM.Subnet = opt.ifInfo.HostSubnetPrefix.String() // populate endpoint info section @@ -684,23 +683,20 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) // populate endpoint info - // populate endpoint info fields code is below - //defaultInterfaceInfo := opt.ifInfo epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, opt.ifInfo.DNS, opt.k8sNamespace) // Probably won't panic if given bad values if err != nil { - // TODO: will it error out if we have a secondary endpoint that has blank DNS though? If so, problem! Discussed and answer is no. + err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) return nil, err } policyArgs := PolicyArgs{ - // pass podsubnet info etc. part of epinfo - subnetInfos: nwInfo.Subnets, // TODO: (1/2 opt.nwInfo) we do not have the full nw info created yet-- is this okay? Discussed and answer is it's fine. getEndpointPolicies requires nwInfo.Subnets only (checked) + subnetInfos: nwInfo.Subnets, // getEndpointPolicies requires nwInfo.Subnets only (checked) nwCfg: opt.nwCfg, ipconfigs: opt.ifInfo.IPConfigs, } endpointPolicies, err := getEndpointPolicies(policyArgs) if err != nil { - // TODO: should only error out if we have an ip config and it is not readable. Discussed and answer is it's fine. + logger.Error("Failed to get endpoint policies", zap.Error(err)) return nil, err } @@ -712,7 +708,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. // IT will result in unpredictable behavior if API server decides to // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", nwInfo.NetworkId, opt.args.ContainerID, opt.args.IfName) // TODO: (2/2 opt.nwInfo) We use the nwInfo we generated above + vethName = fmt.Sprintf("%s%s%s", nwInfo.NetworkId, opt.args.ContainerID, opt.args.IfName) } // for secondary (Populate addresses) @@ -760,13 +756,11 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn if opt.ipamAddResult.ipv6Enabled { // not specific to this particular interface epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working } - // TODO: REMOVE (any azIpam or azIpamResult.IPs should be removed, or infra vnet ip or epInfo.InfraVnetIP-- don't remove from endpoint or network state) - // PROBLEM: The infra vnet ip is used in OVS etc. (snat rules) + if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address } - // TODO: Do I remove azIpamResult as a param from the function below? It is used in linux (changes ep info routes)! if opt.nwCfg.MultiTenancy { // previously only infra nic was passed into this function but now all nics are passed in (possibly breaks swift v2) plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, opt.ifInfo) diff --git a/network/manager_mock.go b/network/manager_mock.go index 8cc1ab3c3a..ea96800970 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -2,6 +2,7 @@ package network import ( "github.com/Azure/azure-container-networking/common" + "github.com/pkg/errors" ) // MockNetworkManager is a mock structure for Network Manager @@ -175,8 +176,7 @@ func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*Endp ep, err := nm.CreateEndpoint(client, epInfo.NetworkId, epInfo) if err != nil { - // err = plugin.Errorf("Failed to create endpoint: %v", err) - return err //added + return errors.Wrap(err, "Failed to create endpoint") } eps = append(eps, ep) } diff --git a/network/network.go b/network/network.go index 3b126f9d46..f689c5319d 100644 --- a/network/network.go +++ b/network/network.go @@ -10,6 +10,7 @@ import ( "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" + "github.com/pkg/errors" "go.uber.org/zap" ) @@ -324,8 +325,7 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo // nwInfo.NetworkId is the same as epInfo.NetworkId ep, err := nm.CreateEndpoint(cnsclient, epInfo.NetworkId, epInfo) if err != nil { - // err = plugin.Errorf("Failed to create endpoint: %v", err) - return err //added + return errors.Wrap(err, "Failed to create endpoint") } eps = append(eps, ep) From ea5b8066b99089123a06e27ce5a2f368e7e17045 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 8 Apr 2024 16:50:35 -0700 Subject: [PATCH 045/102] make invoker_cns_test linux pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit running all linux package tests for network and cni package pass (or also fail on master, like createBridge) windows unit tests mostly all fail for the same ones on master and this branch summary: - network_windows_test.go â—‹ TestFailToAddIPv6DefaultRoute already fails on master - network_test.go â—‹ 9 tests fail on master, 9 tests fail on my branch - manager_test.go â—‹ 9 tests fail on master, 9 tests fail on my branch - endpoint_windows_test.go â—‹ TestNewAndDeleteEndpointImplHnsV2 already timeouts on master - endpoint_test.go â—‹ 9 tests fail on master, 9 tests fail on my branch - network_windows_test.go â—‹ FAIL: TestPluginSecondAddSamePodWindows/CNI_consecutive_add_already_hot_attached â—‹ FAIL: TestPluginSecondAddSamePodWindows/CNI_consecutive_add_not_hot_attached â—‹ We don't handle consecutive add anymore - network_test.go â—‹ TestPluginMultitenancyAdd/Add_Happy_path fails on master and my branch (received multiple NC results [] from CNS while dualnic feature is not supported)-- we still get two items on our list/map though which is expected - invoker_cns_test passes - invoker_azure_test passes - multitenancy_test passes The consecutive add tests fail but that is expected since we no longer support it. --- cni/network/invoker_azure_test.go | 12 ++++++++++++ cni/network/invoker_cns.go | 2 ++ cni/network/invoker_cns_test.go | 16 +++++++++++----- cni/network/network.go | 5 ++--- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index fae03b1b44..26b32fe514 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -82,6 +82,9 @@ func (m *mockDelegatePlugin) Errorf(format string, args ...interface{}) *cniType } } +// net.ParseCIDR will first get the ip, which contains byte data for the ip and mask, +// and the ipnet, which has a field for the *masked* ip and a field for the mask +// this function then replaces the masked ip with the "ip" field retrieved earlier and returns the ipnet func getCIDRNotationForAddress(ipaddresswithcidr string) *net.IPNet { ip, ipnet, err := net.ParseCIDR(ipaddresswithcidr) if err != nil { @@ -91,6 +94,15 @@ func getCIDRNotationForAddress(ipaddresswithcidr string) *net.IPNet { return ipnet } +// returns an ipnet, which contains the *masked* ip (zeroed out based on CIDR) and the mask itself +func parseCIDR(ipaddresswithcidr string) *net.IPNet { + _, ipnet, err := net.ParseCIDR(ipaddresswithcidr) + if err != nil { + panic(fmt.Sprintf("failed to parse cidr with err: %v", err)) + } + return ipnet +} + func getSingleResult(ip string) []*cniTypesCurr.Result { return []*cniTypesCurr.Result{ { diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index c29b76bcef..ad471da7ba 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -430,6 +430,8 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add Gw: ncgw, }) } + // if we have multiple infra ip result infos, we effectively append routes and ip configs to that same interface info each time + // the host subnet prefix (in ipv4 or ipv6) will always refer to the same interface regardless of which ip result info we look at addResult.interfaceInfo[string(info.nicType)] = network.InterfaceInfo{ NICType: cns.InfraNIC, SkipDefaultRoutes: info.skipDefaultRoutes, diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index ea11f1aa3e..a6858c6b0e 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -142,7 +142,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { }, }, NICType: cns.InfraNIC, - HostSubnetPrefix: *getCIDRNotationForAddress("10.224.0.0/16"), + HostSubnetPrefix: *parseCIDR("10.224.0.0/16"), }, wantErr: false, }, @@ -239,7 +239,8 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { Gw: net.ParseIP("fe80::1234:5678:9abc"), }, }, - NICType: cns.InfraNIC, + NICType: cns.InfraNIC, + HostSubnetPrefix: *parseCIDR("fd11:1234::/112"), }, wantErr: false, }, @@ -323,6 +324,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { }, NICType: cns.InfraNIC, SkipDefaultRoutes: true, + HostSubnetPrefix: *parseCIDR("10.0.0.0/24"), }, wantSecondaryInterfacesInfo: network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ @@ -333,6 +335,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { Routes: []network.RouteInfo{}, NICType: cns.DelegatedVMNIC, MacAddress: parsedMacAddress, + // secondaries don't have a host subnet prefix }, wantErr: false, }, @@ -592,7 +595,8 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { Gw: net.ParseIP("10.0.0.1"), }, }, - NICType: cns.InfraNIC, + NICType: cns.InfraNIC, + HostSubnetPrefix: *parseCIDR("10.0.0.0/24"), }, wantErr: false, }, @@ -688,7 +692,8 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { Gw: net.ParseIP("fe80::1234:5678:9abc"), }, }, - NICType: cns.InfraNIC, + NICType: cns.InfraNIC, + HostSubnetPrefix: *parseCIDR("fd11:1234::/112"), }, wantErr: false, }, @@ -832,7 +837,8 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { Gw: net.ParseIP("10.0.0.1"), }, }, - NICType: cns.InfraNIC, + NICType: cns.InfraNIC, + HostSubnetPrefix: *parseCIDR("10.0.0.0/24"), }, wantErr: true, }, diff --git a/cni/network/network.go b/cni/network/network.go index c3b714bf25..ca1581a9ca 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -359,7 +359,6 @@ func (plugin *NetPlugin) getNetworkInfo(netNs string, interfaceInfo *network.Int func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { var ( ipamAddResult IPAMAddResult - ipamAddResults []IPAMAddResult azIpamResult *cniTypesCurr.Result enableInfraVnet bool enableSnatForDNS bool @@ -507,9 +506,9 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } // dual nic when we get multiple interface infos back (multitenancy does not necessarily have multiple if infos) if len(ipamAddResult.interfaceInfo) > 1 && !plugin.isDualNicFeatureSupported(args.Netns) { - errMsg := fmt.Sprintf("received multiple NC results %+v from CNS while dualnic feature is not supported", ipamAddResults) + errMsg := fmt.Sprintf("received multiple NC results %+v from CNS while dualnic feature is not supported", ipamAddResult.interfaceInfo) logger.Error("received multiple NC results from CNS while dualnic feature is not supported", - zap.Any("results", ipamAddResult)) + zap.Any("results", ipamAddResult.interfaceInfo)) return plugin.Errorf(errMsg) } } else { From b2f2954247cec747464ff1dc8711849b5232e1bf Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 12 Apr 2024 11:32:24 -0700 Subject: [PATCH 046/102] modify delete flow to handle multiple epinfos to delete delete ALL endpoints related to the endpoint infos list in the event cni fails half-way through an add (one failed endpoint create and we delete all would-be-create endpoints and the state) replace looping over deletion code "n" number of times with getting a slice of endpoint infos to delete modify stateless cni code to retrieve a slice of network endpoint infos from a single response based on the container id (container id can be used in stateless cni for retrieval) incorporate stateless cni changes from other branch (cns client/ipam/restserver changes) modify get endpoint state to return slice of endpoint infos, and getting an endpoint will return an endpoint from that slice with nic type infra move edge case where endpoint is not created in the state but ips are already allocated to immediately after retrieving all ep infos fix mock behavior for getting all endpoints by container id move getting network id and network info out of the loop because their values do not seem to change between iterations move deletion of endpoint logic into a dedicated loop, and then create a dedicate loop for calling ipam delete to prevent inconsistent state all expected unit tests on linux pass --- cni/network/network.go | 184 ++++++++++++++--------- cns/api.go | 1 + cns/client/client.go | 9 +- cns/client/client_test.go | 14 +- cns/restserver/ipam.go | 15 +- cns/restserver/restserver.go | 9 +- network/endpoint.go | 1 + network/manager.go | 130 +++++++++++----- network/manager_mock.go | 17 +++ network/network.go | 7 +- network/ovs_endpoint_infraroute_linux.go | 2 +- network/ovs_endpointclient_linux.go | 2 +- 12 files changed, 265 insertions(+), 126 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index ca1581a9ca..3028061fb2 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -486,6 +486,8 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} if nwCfg.MultiTenancy { + // dual nic multitenancy -> two interface infos + // multitenancy (swift v1) -> one interface info plugin.report.Context = "AzureCNIMultitenancy" plugin.multitenancyClient.Init(cnsClient, AzureNetIOShim{}) @@ -512,6 +514,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { return plugin.Errorf(errMsg) } } else { + // when nwcfg.multitenancy (use multitenancy flag for swift v1 only) is false if plugin.ipamInvoker == nil { switch nwCfg.IPAM.Type { case network.AzureCNS: @@ -532,7 +535,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) } - endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) // moved to addIpamInvoker // sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam interface: %+v", ipamAddResult.PrettyString())) @@ -540,6 +542,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { defer func() { //nolint:gocritic if err != nil { // for multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips + // if nwCfg.Multitenancy is false or ipam type is not azure cns if !(nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS) { for _, ifInfo := range ipamAddResult.interfaceInfo { // This used to only be called for infraNIC, test if this breaks scenarios @@ -553,10 +556,13 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { }() epInfos := []*network.EndpointInfo{} + var infraSeen bool = false + i := 0 for _, ifInfo := range ipamAddResult.interfaceInfo { // TODO: hopefully I can get natInfo here? natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) networkID, _ := plugin.getNetworkID(args.Netns, &ifInfo, nwCfg) + createEpInfoOpt := createEpInfoOpt{ nwCfg: nwCfg, cnsNetworkConfig: ifInfo.NCResponse, @@ -564,7 +570,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { azIpamResult: azIpamResult, args: args, policies: policies, - endpointID: endpointID, k8sPodName: k8sPodName, k8sNamespace: k8sNamespace, enableInfraVnet: enableInfraVnet, @@ -575,6 +580,9 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { ifInfo: &ifInfo, ipamAddConfig: &ipamAddConfig, ipv6Enabled: ipamAddResult.ipv6Enabled, + + infraSeen: &infraSeen, + idx: i, } var epInfo *network.EndpointInfo epInfo, err = plugin.createEpInfo(&createEpInfoOpt) @@ -585,12 +593,31 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // TODO: should this statement be based on the current iteration instead of the constant ifIndex? // TODO figure out where to put telemetry: sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", // ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) - + i += 1 } cnsclient, err := cnscli.New(nwCfg.CNSUrl, defaultRequestTimeout) if err != nil { return errors.Wrap(err, "failed to create cns client") } + defer func() { + if err != nil { + // CHECK: can we just keep going through each ep info ignoring if the ep actually exists? + // Delete all endpoints + for _, epInfo := range epInfos { + deleteErr := plugin.nm.DeleteEndpoint(epInfo.NetworkId, epInfo.EndpointID, epInfo) + if deleteErr != nil { + logger.Error("Could not delete endpoint after detecting add failure", zap.String("epInfo", epInfo.PrettyString())) + } + } + // Rely on cleanupAllocationOnError declared above to delete ips + // Delete state in disk here + delErr := plugin.nm.DeleteState(epInfos) + if delErr != nil { + logger.Error("Could not delete state after detecting add failure") + } + } + }() + // CHECK: why does the cns client deal with apipa?-- "cns client" only used in windows to create apipa err = plugin.nm.EndpointCreate(cnsclient, epInfos) if err != nil { return err // behavior can change if you don't assign to err prior to returning @@ -619,7 +646,6 @@ type createEpInfoOpt struct { azIpamResult *cniTypesCurr.Result args *cniSkel.CmdArgs policies []policy.Policy - endpointID string k8sPodName string k8sNamespace string enableInfraVnet bool @@ -630,6 +656,9 @@ type createEpInfoOpt struct { ifInfo *network.InterfaceInfo ipamAddConfig *IPAMAddConfig ipv6Enabled bool + + infraSeen *bool // Only the first infra gets args.ifName, even if the second infra is on a different network + idx int } func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointInfo, error) { // you can modify to pass in whatever else you need @@ -717,8 +746,18 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn addresses = append(addresses, ipconfig.Address) } + // generate endpoint info + var endpointID string + if opt.ifInfo.NICType == cns.InfraNIC && !*opt.infraSeen { + // so we do not break existing scenarios, only the first infra gets the original endpoint id generation + endpointID = plugin.nm.GetEndpointID(opt.args.ContainerID, opt.args.IfName) + *opt.infraSeen = true + } else { + endpointID = plugin.nm.GetEndpointID(opt.args.ContainerID, strconv.Itoa(opt.idx)) + } + epInfo = network.EndpointInfo{ - EndpointID: opt.endpointID, + EndpointID: endpointID, ContainerID: opt.args.ContainerID, NetNsPath: opt.args.Netns, IfName: opt.args.IfName, @@ -951,8 +990,6 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { k8sPodName string k8sNamespace string networkID string - nwInfo network.EndpointInfo - epInfo *network.EndpointInfo cniMetric telemetry.AIMetric ) @@ -1028,7 +1065,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) default: - plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) + plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &network.EndpointInfo{}) } } @@ -1037,81 +1074,89 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { // deleted, getNetworkName will return error of the type NetworkNotFoundError which will result in nil error as compliance // with CNI SPEC as mentioned below. - numEndpointsToDelete := 1 - // only get number of endpoints if it's multitenancy mode - if nwCfg.MultiTenancy { - numEndpointsToDelete = plugin.nm.GetNumEndpointsByContainerID(args.ContainerID) - } - - logger.Info("Endpoints to be deleted", zap.Int("count", numEndpointsToDelete)) - for i := 0; i < numEndpointsToDelete; i++ { - // Initialize values from network config. - networkID, err = plugin.getNetworkName(args.Netns, nil, nwCfg) - if err != nil { - // If error is not found error, then we ignore it, to comply with CNI SPEC. - if network.IsNetworkNotFoundError(err) { - err = nil + // CHECK: args.Netns or epInfo.Netns (it's not populated when we convert ep to epInfo) + // CHECK: When do we have multiple network ids? + networkID, err = plugin.getNetworkID(args.Netns, nil, nwCfg) + // CHECK: When do we get multiple network infos + var nwInfo network.EndpointInfo + if nwInfo, err = plugin.nm.GetNetworkInfo(networkID); err != nil { + if !nwCfg.MultiTenancy { + logger.Error("Failed to query network", + zap.String("network", networkID), + zap.Error(err)) + // Log the error but return success if the network is not found. + // if cni hits this, mostly state file would be missing and it can be reboot scenario where + // container runtime tries to delete and create pods which existed before reboot. + // this condition will not apply to stateless CNI since the network struct will be crated on each call + err = nil + if !plugin.nm.IsStatelessCNIMode() { return err } + } + } - logger.Error("Failed to extract network name from network config", zap.Error(err)) - err = plugin.Errorf("Failed to extract network name from network config. error: %v", err) + // Initialize values from network config. + if err != nil { + // If error is not found error, then we ignore it, to comply with CNI SPEC. + if network.IsNetworkNotFoundError(err) { + err = nil return err } - // Query the network. - if nwInfo, err = plugin.nm.GetNetworkInfo(networkID); err != nil { - if !nwCfg.MultiTenancy { - logger.Error("Failed to query network", - zap.String("network", networkID), - zap.Error(err)) - // Log the error but return success if the network is not found. - // if cni hits this, mostly state file would be missing and it can be reboot scenario where - // container runtime tries to delete and create pods which existed before reboot. - // this condition will not apply to stateless CNI since the network struct will be crated on each call - err = nil - if !plugin.nm.IsStatelessCNIMode() { - return err - } - } - } + logger.Error("Failed to extract network name from network config", zap.Error(err)) + err = plugin.Errorf("Failed to extract network name from network config. error: %v", err) + return err + } + logger.Info("Retrieved network info, populating endpoint infos with container id", zap.String("containerID", args.ContainerID)) + + var epInfos []*network.EndpointInfo + if plugin.nm.IsStatelessCNIMode() { + epInfos, err = plugin.nm.GetEndpointState(networkID, args.ContainerID) + } else { + epInfos = plugin.nm.GetEndpointInfosFromContainerID(args.ContainerID) + } + + // for when the endpoint is not created, but the ips are already allocated (only works if single network, single infra) + // stateless cni won't have this issue + if !plugin.nm.IsStatelessCNIMode() && len(epInfos) == 0 { endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) - // Query the endpoint. - if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID); err != nil { - logger.Info("GetEndpoint", + if !nwCfg.MultiTenancy { + logger.Error("Failed to query endpoint", zap.String("endpoint", endpointID), zap.Error(err)) - if !nwCfg.MultiTenancy { - // attempt to release address associated with this Endpoint id - // This is to ensure clean up is done even in failure cases - - logger.Error("Failed to query endpoint", - zap.String("endpoint", endpointID), - zap.Error(err)) - logger.Error("Release ip by ContainerID (endpoint not found)", - zap.String("containerID", args.ContainerID)) - sendEvent(plugin, fmt.Sprintf("Release ip by ContainerID (endpoint not found):%v", args.ContainerID)) - if err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options); err != nil { - return plugin.RetriableError(fmt.Errorf("failed to release address(no endpoint): %w", err)) - } + logger.Error("Release ip by ContainerID (endpoint not found)", + zap.String("containerID", args.ContainerID)) + sendEvent(plugin, fmt.Sprintf("Release ip by ContainerID (endpoint not found):%v", args.ContainerID)) + if err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options); err != nil { + return plugin.RetriableError(fmt.Errorf("failed to release address(no endpoint): %w", err)) } - // Log the error but return success if the endpoint being deleted is not found. - err = nil - return err + } else { + // TODO: how do we clean up ips in multitenancy case? } - - // schedule send metric before attempting delete - defer sendMetricFunc() //nolint:gocritic - logger.Info("Deleting endpoint", - zap.String("endpointID", endpointID)) - sendEvent(plugin, fmt.Sprintf("Deleting endpoint:%v", endpointID)) - // Delete the endpoint. - if err = plugin.nm.DeleteEndpoint(networkID, endpointID, epInfo); err != nil { + // Log the error but return success if the endpoint being deleted is not found. + err = nil + return err + } + logger.Info("Deleting the endpoints", zap.Any("endpointInfos", epInfos)) + // populate ep infos here in loop if necessary + // delete endpoints + for _, epInfo := range epInfos { + // CHECK: networkID or epInfo.networkID (it's not populated when we convert ep to epInfo) + if err = plugin.nm.DeleteEndpoint(networkID, epInfo.EndpointID, epInfo); err != nil { // return a retriable error so the container runtime will retry this DEL later // the implementation of this function returns nil if the endpoint doens't exist, so // we don't have to check that here return plugin.RetriableError(fmt.Errorf("failed to delete endpoint: %w", err)) } + } + logger.Info("Deleting the endpoints from the ipam") + // delete endpoint state in cns and in statefile + for _, epInfo := range epInfos { + // schedule send metric before attempting delete + defer sendMetricFunc() //nolint:gocritic + logger.Info("Deleting endpoint", + zap.String("endpointID", epInfo.EndpointID)) + sendEvent(plugin, fmt.Sprintf("Deleting endpoint:%v", epInfo.EndpointID)) if !nwCfg.MultiTenancy { // Call into IPAM plugin to release the endpoint's addresses. @@ -1132,6 +1177,11 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { } } } + logger.Info("Deleting the state from the cni statefile") + err = plugin.nm.DeleteState(epInfos) + if err != nil { + return plugin.RetriableError(fmt.Errorf("failed to save state: %w", err)) + } sendEvent(plugin, fmt.Sprintf("CNI DEL succeeded : Released ip %+v podname %v namespace %v", nwCfg.IPAM.Address, k8sPodName, k8sNamespace)) return err diff --git a/cns/api.go b/cns/api.go index 3894e9aff6..39d5a23a87 100644 --- a/cns/api.go +++ b/cns/api.go @@ -363,4 +363,5 @@ type GetHomeAzResponse struct { type EndpointRequest struct { HnsEndpointID string `json:"hnsEndpointID"` HostVethName string `json:"hostVethName"` + InterfaceName string `json:"InterfaceName"` } diff --git a/cns/client/client.go b/cns/client/client.go index 2ee524cfb9..5397fc0660 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1023,11 +1023,11 @@ func (c *Client) GetHomeAz(ctx context.Context) (*cns.GetHomeAzResponse, error) return &getHomeAzResponse, nil } -// GetEndpoint calls the EndpointHandlerAPI in CNS to retrieve the state of a given EndpointID -func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*restserver.GetEndpointResponse, error) { +// GetEndpoint calls the EndpointHandlerAPI in CNS to retrieve the state of a given containerID +func (c *Client) GetEndpoint(ctx context.Context, containerID string) (*restserver.GetEndpointResponse, error) { // build the request u := c.routes[cns.EndpointAPI] - uString := u.String() + endpointID + uString := u.String() + containerID req, err := http.NewRequestWithContext(ctx, http.MethodGet, uString, http.NoBody) if err != nil { return nil, errors.Wrap(err, "failed to build request") @@ -1059,11 +1059,12 @@ func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*restserve // UpdateEndpoint calls the EndpointHandlerAPI in CNS // to update the state of a given EndpointID with either HNSEndpointID or HostVethName -func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName string) (*cns.Response, error) { +func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName string, ifName string) (*cns.Response, error) { // build the request updateEndpoint := cns.EndpointRequest{ HnsEndpointID: hnsID, HostVethName: vethName, + InterfaceName: ifName, } var body bytes.Buffer diff --git a/cns/client/client_test.go b/cns/client/client_test.go index 65c427c844..17bcb724db 100644 --- a/cns/client/client_test.go +++ b/cns/client/client_test.go @@ -2710,6 +2710,7 @@ func TestUpdateEndpoint(t *testing.T) { containerID string hnsID string vethName string + ifName string response *RequestCapture expReq *cns.EndpointRequest shouldErr bool @@ -2719,6 +2720,7 @@ func TestUpdateEndpoint(t *testing.T) { "", "", "", + "", &RequestCapture{ Next: &mockdo{}, }, @@ -2730,6 +2732,7 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "bar", "", + "eth0", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, @@ -2737,6 +2740,7 @@ func TestUpdateEndpoint(t *testing.T) { }, &cns.EndpointRequest{ HnsEndpointID: "bar", + InterfaceName: "eth0", }, false, }, @@ -2745,13 +2749,15 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "", "bar", + "eth0", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, }, }, &cns.EndpointRequest{ - HostVethName: "bar", + HostVethName: "bar", + InterfaceName: "eth0", }, false, }, @@ -2760,13 +2766,15 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "", "bar", + "eth0", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusBadRequest, }, }, &cns.EndpointRequest{ - HostVethName: "bar", + HostVethName: "bar", + InterfaceName: "eth0", }, true, }, @@ -2784,7 +2792,7 @@ func TestUpdateEndpoint(t *testing.T) { } // execute the method under test - res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName) + res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName, test.ifName) if err != nil && !test.shouldErr { t.Fatal("unexpected error: err: ", err, res.Message) } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 2e1115039c..bb4459775f 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -1109,6 +1109,17 @@ func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r * logger.Response(service.Name, response, response.ReturnCode, err) return } + if req.InterfaceName == "" { + logger.Warnf("[updateEndpoint] No Interface has been provided") + response := cns.Response{ + ReturnCode: types.InvalidRequest, + Message: "[updateEndpoint] No Interface has been provided", + } + w.Header().Set(cnsReturnCode, response.ReturnCode.String()) + err = service.Listener.Encode(w, &response) + logger.Response(service.Name, response, response.ReturnCode, err) + return + } // Update the endpoint state err = service.UpdateEndpointHelper(endpointID, req) if err != nil { @@ -1139,11 +1150,11 @@ func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req cns. if endpointInfo, ok := service.EndpointState[endpointID]; ok { logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s", endpointID) if req.HnsEndpointID != "" { - service.EndpointState[endpointID].HnsEndpointID = req.HnsEndpointID + service.EndpointState[endpointID].IfnameToIPMap[req.InterfaceName].HnsEndpointID = req.HnsEndpointID logger.Printf("[updateEndpoint] update the endpoint %s with HNSID %s", endpointID, req.HnsEndpointID) } if req.HostVethName != "" { - service.EndpointState[endpointID].HostVethName = req.HostVethName + service.EndpointState[endpointID].IfnameToIPMap[req.InterfaceName].HostVethName = req.HostVethName logger.Printf("[updateEndpoint] update the endpoint %s with vethName %s", endpointID, req.HostVethName) } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index d8e9604a7b..fc54046839 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -93,13 +93,14 @@ type EndpointInfo struct { PodName string PodNamespace string IfnameToIPMap map[string]*IPInfo // key : interface name, value : IPInfo - HnsEndpointID string - HostVethName string } type IPInfo struct { - IPv4 []net.IPNet - IPv6 []net.IPNet + IPv4 []net.IPNet + IPv6 []net.IPNet + HnsEndpointID string + HostVethName string + NICType cns.NICType } type GetHTTPServiceDataResponse struct { diff --git a/network/endpoint.go b/network/endpoint.go index 6cb5228580..070d29c249 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -299,6 +299,7 @@ func (ep *endpoint) getInfo() *EndpointInfo { NetworkContainerID: ep.NetworkContainerID, HNSEndpointID: ep.HnsId, HostIfName: ep.HostIfName, + NICType: ep.NICType, // CHECK: Any more fields to convert endpoint into endpoint info? } info.Routes = append(info.Routes, ep.Routes...) diff --git a/network/manager.go b/network/manager.go index 31bea5f573..6374b1dc92 100644 --- a/network/manager.go +++ b/network/manager.go @@ -9,7 +9,9 @@ import ( "sync" "time" + "github.com/Azure/azure-container-networking/cns" cnsclient "github.com/Azure/azure-container-networking/cns/client" + "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -70,7 +72,7 @@ type EndpointClient interface { // NetworkManager manages the set of container networking resources. type networkManager struct { - StatelessCniMode bool + statelessCniMode bool CnsClient *cnsclient.Client Version string TimeStamp time.Time @@ -111,6 +113,9 @@ type NetworkManager interface { GetEndpointID(containerID, ifName string) string IsStatelessCNIMode() bool SaveState(eps []*endpoint) error + DeleteState(epInfos []*EndpointInfo) error + GetEndpointInfosFromContainerID(containerID string) []*EndpointInfo + GetEndpointState(networkID, containerID string) ([]*EndpointInfo, error) } // Creates a new network manager. @@ -151,7 +156,7 @@ func (nm *networkManager) Uninitialize() { // SetStatelessCNIMode enable the statelessCNI falg and inititlizes a CNSClient func (nm *networkManager) SetStatelessCNIMode() error { - nm.StatelessCniMode = true + nm.statelessCniMode = true // Create CNS client client, err := cnsclient.New(cnsBaseURL, cnsReqTimeout) if err != nil { @@ -163,7 +168,7 @@ func (nm *networkManager) SetStatelessCNIMode() error { // IsStatelessCNIMode checks if the Stateless CNI mode has been enabled or not func (nm *networkManager) IsStatelessCNIMode() bool { - return nm.StatelessCniMode + return nm.statelessCniMode } // Restore reads network manager state from persistent store. @@ -403,7 +408,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn // It will add HNSEndpointID or HostVeth name to the endpoint state func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { logger.Info("Calling cns updateEndpoint API with ", zap.String("containerID: ", ep.ContainerID), zap.String("HnsId: ", ep.HnsId), zap.String("HostIfName: ", ep.HostIfName)) - response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName) + response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName, ep.IfName) if err != nil { return errors.Wrapf(err, "Update endpoint API returend with error") } @@ -413,36 +418,25 @@ func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { // GetEndpointState will make a call to CNS GetEndpointState API in the stateless CNI mode to fetch the endpointInfo // TODO unit tests need to be added, WorkItem: 26606939 -func (nm *networkManager) GetEndpointState(networkID, endpointID string) (*EndpointInfo, error) { - endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), endpointID) +// In stateless cni, container id is the endpoint id, so you can pass in either +func (nm *networkManager) GetEndpointState(networkID, containerID string) ([]*EndpointInfo, error) { + endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), containerID) if err != nil { - return nil, errors.Wrapf(err, "Get endpoint API returend with error") + return nil, errors.Wrapf(err, "Get endpoint API returned with error") } - epInfo := &EndpointInfo{ - EndpointID: endpointID, - IfIndex: EndpointIfIndex, // Azure CNI supports only one interface - IfName: endpointResponse.EndpointInfo.HostVethName, - ContainerID: endpointID, - PODName: endpointResponse.EndpointInfo.PodName, - PODNameSpace: endpointResponse.EndpointInfo.PodNamespace, - NetworkContainerID: endpointID, - HNSEndpointID: endpointResponse.EndpointInfo.HnsEndpointID, - // TODO: Populate NIC Type - } - - for _, ip := range endpointResponse.EndpointInfo.IfnameToIPMap { - epInfo.IPAddresses = ip.IPv4 - epInfo.IPAddresses = append(epInfo.IPAddresses, ip.IPv6...) + epInfos := cnsEndpointInfotoCNIEpInfos(endpointResponse.EndpointInfo, containerID) - } - if epInfo.IsEndpointStateIncomplete() { - epInfo, err = epInfo.GetEndpointInfoByIPImpl(epInfo.IPAddresses, networkID) - if err != nil { - return nil, errors.Wrapf(err, "Get endpoint API returend with error") + for i := 0; i < len(epInfos); i++ { + if epInfos[i].NICType == cns.InfraNIC { + if epInfos[i].IsEndpointStateIncomplete() { + epInfos[i], err = epInfos[i].GetEndpointInfoByIPImpl(epInfos[i].IPAddresses, networkID) + if err != nil { + return nil, errors.Wrapf(err, "Get endpoint API returned with error") + } + } } } - logger.Info("returning getEndpoint API with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", epInfo.HNSEndpointID)) - return epInfo, nil + return epInfos, nil } // DeleteEndpoint deletes an existing container endpoint. @@ -451,6 +445,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E defer nm.Unlock() if nm.IsStatelessCNIMode() { + // CHECK: Performs same actions as below, but creates the endpoint in a different way (doesn't actually make calls to cns) return nm.DeleteEndpointState(networkID, epInfo) } @@ -464,11 +459,6 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E return err } - err = nm.save() - if err != nil { - return err - } - return nil } @@ -507,9 +497,17 @@ func (nm *networkManager) GetEndpointInfo(networkID, endpointID string) (*Endpoi if nm.IsStatelessCNIMode() { logger.Info("calling cns getEndpoint API") - epInfo, err := nm.GetEndpointState(networkID, endpointID) - - return epInfo, err + epInfos, err := nm.GetEndpointState(networkID, endpointID) + if err != nil { + return nil, err + } + for _, epInfo := range epInfos { + if epInfo.NICType == cns.InfraNIC { + return epInfo, nil + } + } + // CHECK: what if multiple infra nics or no infra nics? Guess: multi infra nics are interchangeable + return nil, err } nw, err := nm.getNetwork(networkID) @@ -671,7 +669,7 @@ func (nm *networkManager) GetNumberOfEndpoints(ifName string, networkId string) // GetEndpointID returns a unique endpoint ID based on the CNI mode. func (nm *networkManager) GetEndpointID(containerID, ifName string) string { if nm.IsStatelessCNIMode() { - return containerID + return containerID // CHECK: In stateless cni do we need to also add the num or is it always the same ep id for all endpoints with the same container id? } if len(containerID) > ContainerIDLength { containerID = containerID[:ContainerIDLength] @@ -688,7 +686,7 @@ func (nm *networkManager) SaveState(eps []*endpoint) error { defer nm.Unlock() logger.Info("Saving state") - // TODO: What if we fail halfway through? + // If we fail half way, we'll propagate an error up which should clean everything up for _, ep := range eps { if nm.IsStatelessCNIMode() { err := nm.UpdateEndpointState(ep) @@ -705,3 +703,57 @@ func (nm *networkManager) SaveState(eps []*endpoint) error { // once endpoints and networks are in-memory, save once return nm.save() } +func (nm *networkManager) DeleteState(epInfos []*EndpointInfo) error { + nm.Lock() + defer nm.Unlock() + + logger.Info("Deleting state") + // We do not use DeleteEndpointState for stateless cni because we already call it in DeleteEndpoint + // This function is only for saving to stateless cni or the cni statefile + // For stateless cni, plugin.ipamInvoker.Delete takes care of removing the state in the main Delete function + + if nm.IsStatelessCNIMode() { + return nil + } + + // once endpoints and networks are deleted in-memory, save once + return nm.save() +} + +// called to convert a cns restserver EndpointInfo into a network EndpointInfo +func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointID string) []*EndpointInfo { + ret := []*EndpointInfo{} + + for ifName, ipInfo := range endpointInfo.IfnameToIPMap { + epInfo := &EndpointInfo{ + EndpointID: endpointID, // endpoint id is always the same, but we shouldn't use it in the stateless path + IfIndex: EndpointIfIndex, // Azure CNI supports only one interface + ContainerID: endpointID, + PODName: endpointInfo.PodName, + PODNameSpace: endpointInfo.PodNamespace, + NetworkContainerID: endpointID, + } + + epInfo.IPAddresses = ipInfo.IPv4 + epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...) + epInfo.IfName = ifName + epInfo.HostIfName = ipInfo.HostVethName + epInfo.HNSEndpointID = ipInfo.HnsEndpointID + epInfo.NICType = ipInfo.NICType + ret = append(ret, epInfo) + } + return ret +} +func (nm *networkManager) GetEndpointInfosFromContainerID(containerID string) []*EndpointInfo { + ret := []*EndpointInfo{} + for _, extIf := range nm.ExternalInterfaces { + for _, nw := range extIf.Networks { + for _, ep := range nw.Endpoints { + if ep.ContainerID == containerID { + ret = append(ret, ep.getInfo()) + } + } + } + } + return ret +} diff --git a/network/manager_mock.go b/network/manager_mock.go index ea96800970..4aacedd823 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -184,3 +184,20 @@ func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*Endp // save endpoints return nm.SaveState(eps) } + +// TODO: understand mock behavior +func (nm *MockNetworkManager) DeleteState(epInfos []*EndpointInfo) error { + return nil +} +func (nm *MockNetworkManager) GetEndpointInfosFromContainerID(containerID string) []*EndpointInfo { + ret := []*EndpointInfo{} + for _, epInfo := range nm.TestEndpointInfoMap { + if epInfo.ContainerID == containerID { + ret = append(ret, epInfo) + } + } + return ret +} +func (nm *MockNetworkManager) GetEndpointState(networkID, containerID string) ([]*EndpointInfo, error) { + return []*EndpointInfo{}, nil +} diff --git a/network/network.go b/network/network.go index f689c5319d..c77efc019d 100644 --- a/network/network.go +++ b/network/network.go @@ -301,7 +301,7 @@ func (nm *networkManager) GetNumEndpointsByContainerID(containerID string) int { // networkID is opt.nwInfo.ID // cns url is opt.nwCfg.CNSUrl -// Creates the network and corresponding endpoint +// Creates the network and corresponding endpoint (should be called once during Add) func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*EndpointInfo) error { eps := []*endpoint{} // save endpoints for stateless @@ -316,13 +316,10 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo if err != nil { // TODO: error messages/handling are different in this file // err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) - return err // added + return err } } - // Create the endpoint. - // sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) - // nwInfo.NetworkId is the same as epInfo.NetworkId ep, err := nm.CreateEndpoint(cnsclient, epInfo.NetworkId, epInfo) if err != nil { return errors.Wrap(err, "Failed to create endpoint") diff --git a/network/ovs_endpoint_infraroute_linux.go b/network/ovs_endpoint_infraroute_linux.go index 829a58ee4c..c1ace0423c 100644 --- a/network/ovs_endpoint_infraroute_linux.go +++ b/network/ovs_endpoint_infraroute_linux.go @@ -62,7 +62,7 @@ func ConfigureInfraVnetContainerInterface(client *OVSEndpointClient, infraIP net return nil } -func DeleteInfraVnetEndpoint(client *OVSEndpointClient, epID string) error { +func DeleteInfraVnetEndpoint(client *OVSEndpointClient) error { if client.enableInfraVnet { return client.infraVnetClient.DeleteInfraVnetEndpoint() } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 97f4186b7d..f82724a3e7 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -251,5 +251,5 @@ func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { if err := client.DeleteSnatEndpoint(); err != nil { return err } - return DeleteInfraVnetEndpoint(client, ep.Id[:7]) + return DeleteInfraVnetEndpoint(client) } From bde67f163057d4df771acad58f37f616e6a3d8be Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 12 Apr 2024 16:10:02 -0700 Subject: [PATCH 047/102] address feedback --- cni/network/network.go | 33 +++++++++++++++++++-------------- network/manager.go | 8 +++++--- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 3028061fb2..737e4c6bac 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -606,14 +606,17 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { for _, epInfo := range epInfos { deleteErr := plugin.nm.DeleteEndpoint(epInfo.NetworkId, epInfo.EndpointID, epInfo) if deleteErr != nil { - logger.Error("Could not delete endpoint after detecting add failure", zap.String("epInfo", epInfo.PrettyString())) + // we already do not return an error when the endpoint is not found, so deleteErr is a real error + logger.Error("Could not delete endpoint after detecting add failure", zap.String("epInfo", epInfo.PrettyString()), zap.Error(deleteErr)) + return } } // Rely on cleanupAllocationOnError declared above to delete ips // Delete state in disk here delErr := plugin.nm.DeleteState(epInfos) if delErr != nil { - logger.Error("Could not delete state after detecting add failure") + logger.Error("Could not delete state after detecting add failure", zap.Error(delErr)) + return } } }() @@ -1094,7 +1097,6 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { } } } - // Initialize values from network config. if err != nil { // If error is not found error, then we ignore it, to comply with CNI SPEC. @@ -1112,26 +1114,27 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { var epInfos []*network.EndpointInfo if plugin.nm.IsStatelessCNIMode() { epInfos, err = plugin.nm.GetEndpointState(networkID, args.ContainerID) + // TODO: epInfos from stateless cni does not have networkID populated and so DeleteEndpoint will not have the networkID } else { epInfos = plugin.nm.GetEndpointInfosFromContainerID(args.ContainerID) } // for when the endpoint is not created, but the ips are already allocated (only works if single network, single infra) // stateless cni won't have this issue - if !plugin.nm.IsStatelessCNIMode() && len(epInfos) == 0 { + if len(epInfos) == 0 { endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) if !nwCfg.MultiTenancy { logger.Error("Failed to query endpoint", zap.String("endpoint", endpointID), zap.Error(err)) - logger.Error("Release ip by ContainerID (endpoint not found)", - zap.String("containerID", args.ContainerID)) - sendEvent(plugin, fmt.Sprintf("Release ip by ContainerID (endpoint not found):%v", args.ContainerID)) - if err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options); err != nil { - return plugin.RetriableError(fmt.Errorf("failed to release address(no endpoint): %w", err)) + if !plugin.nm.IsStatelessCNIMode() { + logger.Error("Release ip by ContainerID (endpoint not found)", + zap.String("containerID", args.ContainerID)) + sendEvent(plugin, fmt.Sprintf("Release ip by ContainerID (endpoint not found):%v", args.ContainerID)) + if err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options); err != nil { + return plugin.RetriableError(fmt.Errorf("failed to release address(no endpoint): %w", err)) + } } - } else { - // TODO: how do we clean up ips in multitenancy case? } // Log the error but return success if the endpoint being deleted is not found. err = nil @@ -1142,7 +1145,8 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { // delete endpoints for _, epInfo := range epInfos { // CHECK: networkID or epInfo.networkID (it's not populated when we convert ep to epInfo) - if err = plugin.nm.DeleteEndpoint(networkID, epInfo.EndpointID, epInfo); err != nil { + if err = plugin.nm.DeleteEndpoint(epInfo.NetworkId, epInfo.EndpointID, epInfo); err != nil { + // An error will not be returned if the endpoint is not found // return a retriable error so the container runtime will retry this DEL later // the implementation of this function returns nil if the endpoint doens't exist, so // we don't have to check that here @@ -1158,7 +1162,8 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { zap.String("endpointID", epInfo.EndpointID)) sendEvent(plugin, fmt.Sprintf("Deleting endpoint:%v", epInfo.EndpointID)) - if !nwCfg.MultiTenancy { + if !nwCfg.MultiTenancy && epInfo.NICType == cns.InfraNIC { + // Delegated/secondary nic ips are statically allocated so we don't need to release // Call into IPAM plugin to release the endpoint's addresses. for i := range epInfo.IPAddresses { logger.Info("Release ip", zap.String("ip", epInfo.IPAddresses[i].IP.String())) @@ -1168,7 +1173,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { return plugin.RetriableError(fmt.Errorf("failed to release address: %w", err)) } } - } else if epInfo.EnableInfraVnet { + } else if epInfo.EnableInfraVnet { // remove in future PR nwCfg.IPAM.Subnet = nwInfo.Subnets[0].Prefix.String() nwCfg.IPAM.Address = epInfo.InfraVnetIP.IP.String() err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options) diff --git a/network/manager.go b/network/manager.go index 6374b1dc92..b5383139c1 100644 --- a/network/manager.go +++ b/network/manager.go @@ -428,7 +428,7 @@ func (nm *networkManager) GetEndpointState(networkID, containerID string) ([]*En for i := 0; i < len(epInfos); i++ { if epInfos[i].NICType == cns.InfraNIC { - if epInfos[i].IsEndpointStateIncomplete() { + if epInfos[i].IsEndpointStateIncomplete() { // assume false for swift v2 epInfos[i], err = epInfos[i].GetEndpointInfoByIPImpl(epInfos[i].IPAddresses, networkID) if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returned with error") @@ -747,10 +747,12 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI func (nm *networkManager) GetEndpointInfosFromContainerID(containerID string) []*EndpointInfo { ret := []*EndpointInfo{} for _, extIf := range nm.ExternalInterfaces { - for _, nw := range extIf.Networks { + for networkID, nw := range extIf.Networks { for _, ep := range nw.Endpoints { if ep.ContainerID == containerID { - ret = append(ret, ep.getInfo()) + val := ep.getInfo() + val.NetworkId = networkID // endpoint doesn't contain the network id + ret = append(ret, val) } } } From 7fc4b643aa3758a0557f2452e83370f4c545a71c Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Fri, 12 Apr 2024 11:06:40 -0700 Subject: [PATCH 048/102] Make change to UpdateEndpointState API to support SwiftV2 for Stateless CNI --- cns/client/client.go | 9 ++----- cns/client/client_test.go | 32 +++++++++++++--------- cns/restserver/ipam.go | 57 ++++++++++++++++++++++----------------- network/manager.go | 19 ++++++++++++- 4 files changed, 71 insertions(+), 46 deletions(-) diff --git a/cns/client/client.go b/cns/client/client.go index 5397fc0660..6815349e8e 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1059,16 +1059,11 @@ func (c *Client) GetEndpoint(ctx context.Context, containerID string) (*restserv // UpdateEndpoint calls the EndpointHandlerAPI in CNS // to update the state of a given EndpointID with either HNSEndpointID or HostVethName -func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName string, ifName string) (*cns.Response, error) { +func (c *Client) UpdateEndpoint(ctx context.Context, endpointID string, ipInfo map[string]*restserver.IPInfo) (*cns.Response, error) { // build the request - updateEndpoint := cns.EndpointRequest{ - HnsEndpointID: hnsID, - HostVethName: vethName, - InterfaceName: ifName, - } var body bytes.Buffer - if err := json.NewEncoder(&body).Encode(updateEndpoint); err != nil { + if err := json.NewEncoder(&body).Encode(ipInfo); err != nil { return nil, errors.Wrap(err, "failed to encode updateEndpoint") } diff --git a/cns/client/client_test.go b/cns/client/client_test.go index 17bcb724db..b10a47f8a5 100644 --- a/cns/client/client_test.go +++ b/cns/client/client_test.go @@ -2712,7 +2712,7 @@ func TestUpdateEndpoint(t *testing.T) { vethName string ifName string response *RequestCapture - expReq *cns.EndpointRequest + expReq map[string]*restserver.IPInfo shouldErr bool }{ { @@ -2738,9 +2738,11 @@ func TestUpdateEndpoint(t *testing.T) { httpStatusCodeToReturn: http.StatusOK, }, }, - &cns.EndpointRequest{ - HnsEndpointID: "bar", - InterfaceName: "eth0", + map[string]*restserver.IPInfo{ + "eth0": { + HnsEndpointID: "bar", + NICType: cns.InfraNIC, + }, }, false, }, @@ -2755,9 +2757,11 @@ func TestUpdateEndpoint(t *testing.T) { httpStatusCodeToReturn: http.StatusOK, }, }, - &cns.EndpointRequest{ - HostVethName: "bar", - InterfaceName: "eth0", + map[string]*restserver.IPInfo{ + "eth0": { + HostVethName: "bar", + NICType: cns.InfraNIC, + }, }, false, }, @@ -2772,9 +2776,11 @@ func TestUpdateEndpoint(t *testing.T) { httpStatusCodeToReturn: http.StatusBadRequest, }, }, - &cns.EndpointRequest{ - HostVethName: "bar", - InterfaceName: "eth0", + map[string]*restserver.IPInfo{ + "eth0": { + HostVethName: "bar", + NICType: cns.InfraNIC, + }, }, true, }, @@ -2792,7 +2798,7 @@ func TestUpdateEndpoint(t *testing.T) { } // execute the method under test - res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName, test.ifName) + res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.expReq) if err != nil && !test.shouldErr { t.Fatal("unexpected error: err: ", err, res.Message) } @@ -2809,7 +2815,7 @@ func TestUpdateEndpoint(t *testing.T) { // if a request was expected to be sent, decode it and ensure that it // matches expectations if test.expReq != nil { - var gotReq cns.EndpointRequest + var gotReq map[string]*restserver.IPInfo err = json.NewDecoder(test.response.Request.Body).Decode(&gotReq) if err != nil { t.Fatal("error decoding the received request: err:", err) @@ -2818,7 +2824,7 @@ func TestUpdateEndpoint(t *testing.T) { // a nil expReq is semantically meaningful (i.e. "no request"), but in // order for cmp to work properly, the outer types should be identical. // Thus we have to dereference it explicitly: - expReq := *test.expReq + expReq := test.expReq // ensure that the received request is what was expected if !cmp.Equal(gotReq, expReq) { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index bb4459775f..420bbb13f4 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -1083,7 +1083,7 @@ func (service *HTTPRestService) GetEndpointHelper(endpointID string) (*EndpointI func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r *http.Request) { logger.Printf("[updateEndpoint] updateEndpoint for %s", r.URL.Path) - var req cns.EndpointRequest + var req map[string]*IPInfo err := service.Listener.Decode(w, r, &req) endpointID := strings.TrimPrefix(r.URL.Path, cns.EndpointPath) logger.Request(service.Name, &req, err) @@ -1098,22 +1098,10 @@ func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r * logger.Response(service.Name, response, response.ReturnCode, err) return } - if req.HostVethName == "" && req.HnsEndpointID == "" { - logger.Warnf("[updateEndpoint] No HnsEndpointID or HostVethName has been provided") + if err = verifyUpdateEndpointStateRequest(req); err != nil { response := cns.Response{ ReturnCode: types.InvalidRequest, - Message: "[updateEndpoint] No HnsEndpointID or HostVethName has been provided", - } - w.Header().Set(cnsReturnCode, response.ReturnCode.String()) - err = service.Listener.Encode(w, &response) - logger.Response(service.Name, response, response.ReturnCode, err) - return - } - if req.InterfaceName == "" { - logger.Warnf("[updateEndpoint] No Interface has been provided") - response := cns.Response{ - ReturnCode: types.InvalidRequest, - Message: "[updateEndpoint] No Interface has been provided", + Message: err.Error(), } w.Header().Set(cnsReturnCode, response.ReturnCode.String()) err = service.Listener.Encode(w, &response) @@ -1142,22 +1130,27 @@ func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r * } // UpdateEndpointHelper updates the state of the given endpointId with HNSId or VethName -func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req cns.EndpointRequest) error { +func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req map[string]*IPInfo) error { if service.EndpointStateStore == nil { return ErrStoreEmpty } logger.Printf("[updateEndpoint] Updating endpoint state for infra container %s", endpointID) if endpointInfo, ok := service.EndpointState[endpointID]; ok { - logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s", endpointID) - if req.HnsEndpointID != "" { - service.EndpointState[endpointID].IfnameToIPMap[req.InterfaceName].HnsEndpointID = req.HnsEndpointID - logger.Printf("[updateEndpoint] update the endpoint %s with HNSID %s", endpointID, req.HnsEndpointID) - } - if req.HostVethName != "" { - service.EndpointState[endpointID].IfnameToIPMap[req.InterfaceName].HostVethName = req.HostVethName - logger.Printf("[updateEndpoint] update the endpoint %s with vethName %s", endpointID, req.HostVethName) + for ifName, InterfaceInfo := range req { + logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s", endpointID) + if InterfaceInfo.HnsEndpointID != "" { + service.EndpointState[endpointID].IfnameToIPMap[ifName].HnsEndpointID = InterfaceInfo.HnsEndpointID + logger.Printf("[updateEndpoint] update the endpoint %s with HNSID %s", endpointID, InterfaceInfo.HnsEndpointID) + } + if InterfaceInfo.HostVethName != "" { + service.EndpointState[endpointID].IfnameToIPMap[ifName].HostVethName = InterfaceInfo.HostVethName + logger.Printf("[updateEndpoint] update the endpoint %s with vethName %s", endpointID, InterfaceInfo.HostVethName) + } + if InterfaceInfo.NICType != "" { + service.EndpointState[endpointID].IfnameToIPMap[ifName].NICType = InterfaceInfo.NICType + logger.Printf("[updateEndpoint] update the endpoint %s with NICType %s", endpointID, InterfaceInfo.NICType) + } } - err := service.EndpointStateStore.Write(EndpointStoreKey, service.EndpointState) if err != nil { return fmt.Errorf("[updateEndpoint] failed to write endpoint state to store for pod %s : %w", endpointInfo.PodName, err) @@ -1165,4 +1158,18 @@ func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req cns. return nil } return errors.New("[updateEndpoint] endpoint could not be found in the statefile") + +} + +// verifyUpdateEndpointStateRequest verify the CNI request body for the UpdateENdpointState API +func verifyUpdateEndpointStateRequest(req map[string]*IPInfo) error { + for ifName, InterfaceInfo := range req { + if InterfaceInfo.HostVethName == "" && InterfaceInfo.HnsEndpointID == "" && InterfaceInfo.NICType == "" { + return errors.New("[updateEndpoint] No NicType, HnsEndpointID or HostVethName has been provided") + } + if ifName == "" { + return errors.New("[updateEndpoint] No Interface has been provided") + } + } + return nil } diff --git a/network/manager.go b/network/manager.go index b5383139c1..019efaef6f 100644 --- a/network/manager.go +++ b/network/manager.go @@ -407,8 +407,9 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn // UpdateEndpointState will make a call to CNS updatEndpointState API in the stateless CNI mode // It will add HNSEndpointID or HostVeth name to the endpoint state func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { + ifnameToIPInfoMap := generateCNSIPInfoMap(ep) // key : interface name, value : IPInfo logger.Info("Calling cns updateEndpoint API with ", zap.String("containerID: ", ep.ContainerID), zap.String("HnsId: ", ep.HnsId), zap.String("HostIfName: ", ep.HostIfName)) - response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName, ep.IfName) + response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ifnameToIPInfoMap) if err != nil { return errors.Wrapf(err, "Update endpoint API returend with error") } @@ -759,3 +760,19 @@ func (nm *networkManager) GetEndpointInfosFromContainerID(containerID string) [] } return ret } + +func generateCNSIPInfoMap(ep *endpoint) map[string]*restserver.IPInfo { + ifNametoIPInfoMap := make(map[string]*restserver.IPInfo) // key : interface name, value : IPInfo + if ep.IfName != "" { + ifNametoIPInfoMap[ep.IfName].NICType = cns.InfraNIC + ifNametoIPInfoMap[ep.IfName].HnsEndpointID = ep.HnsId + ifNametoIPInfoMap[ep.IfName].HostVethName = ep.HostIfName + } + if ep.SecondaryInterfaces != nil { + for ifName, InterfaceInfo := range ep.SecondaryInterfaces { + ifNametoIPInfoMap[ifName].NICType = InterfaceInfo.NICType + } + + } + return ifNametoIPInfoMap +} From 0a56c8909908766f76f36d1f05e541b0c6286e3b Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 12 Apr 2024 16:43:49 -0700 Subject: [PATCH 049/102] change save state to only call update endpoint state once with a slice of endpoints, uts pass --- network/manager.go | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/network/manager.go b/network/manager.go index 019efaef6f..91999a9b3c 100644 --- a/network/manager.go +++ b/network/manager.go @@ -406,10 +406,15 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn // UpdateEndpointState will make a call to CNS updatEndpointState API in the stateless CNI mode // It will add HNSEndpointID or HostVeth name to the endpoint state -func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { - ifnameToIPInfoMap := generateCNSIPInfoMap(ep) // key : interface name, value : IPInfo - logger.Info("Calling cns updateEndpoint API with ", zap.String("containerID: ", ep.ContainerID), zap.String("HnsId: ", ep.HnsId), zap.String("HostIfName: ", ep.HostIfName)) - response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ifnameToIPInfoMap) +func (nm *networkManager) UpdateEndpointState(eps []*endpoint) error { + if len(eps) == 0 { + return nil + } + + ifnameToIPInfoMap := generateCNSIPInfoMap(eps) // key : interface name, value : IPInfo + // logger.Info("Calling cns updateEndpoint API with ", zap.String("containerID: ", ep.ContainerID), zap.String("HnsId: ", ep.HnsId), zap.String("HostIfName: ", ep.HostIfName)) + // we assume all endpoints have the same container id + response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), eps[0].ContainerID, ifnameToIPInfoMap) if err != nil { return errors.Wrapf(err, "Update endpoint API returend with error") } @@ -429,7 +434,7 @@ func (nm *networkManager) GetEndpointState(networkID, containerID string) ([]*En for i := 0; i < len(epInfos); i++ { if epInfos[i].NICType == cns.InfraNIC { - if epInfos[i].IsEndpointStateIncomplete() { // assume false for swift v2 + if epInfos[i].IsEndpointStateIncomplete() { // assume false for swift v2 for now epInfos[i], err = epInfos[i].GetEndpointInfoByIPImpl(epInfos[i].IPAddresses, networkID) if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returned with error") @@ -688,14 +693,13 @@ func (nm *networkManager) SaveState(eps []*endpoint) error { logger.Info("Saving state") // If we fail half way, we'll propagate an error up which should clean everything up - for _, ep := range eps { - if nm.IsStatelessCNIMode() { - err := nm.UpdateEndpointState(ep) - if err != nil { - return err - } + if nm.IsStatelessCNIMode() { + err := nm.UpdateEndpointState(eps) + if err != nil { + return err } } + // we either use stateless cni and save via update endpoint state, or use the state file if nm.IsStatelessCNIMode() { return nil @@ -761,18 +765,16 @@ func (nm *networkManager) GetEndpointInfosFromContainerID(containerID string) [] return ret } -func generateCNSIPInfoMap(ep *endpoint) map[string]*restserver.IPInfo { +func generateCNSIPInfoMap(eps []*endpoint) map[string]*restserver.IPInfo { ifNametoIPInfoMap := make(map[string]*restserver.IPInfo) // key : interface name, value : IPInfo - if ep.IfName != "" { - ifNametoIPInfoMap[ep.IfName].NICType = cns.InfraNIC - ifNametoIPInfoMap[ep.IfName].HnsEndpointID = ep.HnsId - ifNametoIPInfoMap[ep.IfName].HostVethName = ep.HostIfName - } - if ep.SecondaryInterfaces != nil { - for ifName, InterfaceInfo := range ep.SecondaryInterfaces { - ifNametoIPInfoMap[ifName].NICType = InterfaceInfo.NICType - } + for _, ep := range eps { + if ep.IfName != "" { + ifNametoIPInfoMap[ep.IfName].NICType = ep.NICType + ifNametoIPInfoMap[ep.IfName].HnsEndpointID = ep.HnsId + ifNametoIPInfoMap[ep.IfName].HostVethName = ep.HostIfName + } } + return ifNametoIPInfoMap } From 0395e008b1807652b5d169e61b964f70854051a3 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 15 Apr 2024 00:50:13 -0700 Subject: [PATCH 050/102] fix using nonexistent key by passing in current interface info directly --- cni/network/network.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cni/network/network.go b/cni/network/network.go index 737e4c6bac..f3531880ae 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -403,8 +403,10 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // previously just logged the default (infra) interface so this is equivalent behavior cniResult := &cniTypesCurr.Result{} for _, ifInfo := range ipamAddResult.interfaceInfo { + logger.Info("Exiting add, interface info retrieved", zap.Any("ifInfo", ifInfo)) if ifInfo.NICType == cns.InfraNIC { - cniResult = convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[string(cns.InfraNIC)], args.IfName) + cniResult = convertInterfaceInfoToCniResult(ifInfo, args.IfName) + logger.Info("CNI result generated", zap.Any("cniResult", cniResult)) } } From 179b92bbc74e1dbe382abe1a2e5c24c65a4db360 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 15 Apr 2024 17:38:19 -0700 Subject: [PATCH 051/102] fix azure ipam invoker not getting a populated network info for legacy cni --- cni/network/network.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index f3531880ae..3872442cd2 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -995,6 +995,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { k8sPodName string k8sNamespace string networkID string + nwInfo network.EndpointInfo cniMetric telemetry.AIMetric ) @@ -1070,7 +1071,8 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode)) default: - plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &network.EndpointInfo{}) + // nwInfo gets populated later in the function + plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo) } } @@ -1083,7 +1085,6 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { // CHECK: When do we have multiple network ids? networkID, err = plugin.getNetworkID(args.Netns, nil, nwCfg) // CHECK: When do we get multiple network infos - var nwInfo network.EndpointInfo if nwInfo, err = plugin.nm.GetNetworkInfo(networkID); err != nil { if !nwCfg.MultiTenancy { logger.Error("Failed to query network", From 2af154331bf9f2bb8e2aaeaa4edb14862a5d7876 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Tue, 16 Apr 2024 18:48:29 -0400 Subject: [PATCH 052/102] add L1VH windows support --- cni/network/invoker_cns.go | 6 ++++++ network/endpoint_windows.go | 27 +++++++++++++++++++++++++-- network/network_windows.go | 6 ++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 8702e20638..4fd3729334 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -171,6 +171,11 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro numInterfacesWithDefaultRoutes++ } + // Add secondary interface info from podIPInfo to ipamAddResult + info.hostSubnet = response.PodIPInfo[i].HostPrimaryIPInfo.Subnet + info.hostPrimaryIP = response.PodIPInfo[i].HostPrimaryIPInfo.PrimaryIP + info.hostGateway = response.PodIPInfo[i].HostPrimaryIPInfo.Gateway + if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig); err != nil { return IPAMAddResult{}, err } @@ -476,6 +481,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p IP: ip, Mask: ipnet.Mask, }, + Gateway: net.ParseIP(info.ncGatewayIPAddress), }, }, Routes: routes, diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 586c8237bf..cc28a4a56e 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -9,6 +9,7 @@ import ( "net" "strings" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/policy" @@ -75,7 +76,7 @@ func (nw *network) newEndpointImpl( _ ipTablesClient, epInfo *EndpointInfo, ) (*endpoint, error) { - // there is only 1 epInfo for windows, multiple interfaces will be added in the future + if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { if err != nil { return nil, err @@ -85,6 +86,7 @@ func (nw *network) newEndpointImpl( } return nw.newEndpointImplHnsV1(epInfo, plc) + // TODO: add switch statement for NIC type for IB and Accelnet NIC support to create endpoint here in the future } // newEndpointImplHnsV1 creates a new endpoint in the network using HnsV1 @@ -220,9 +222,16 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE Major: hcnSchemaVersionMajor, Minor: hcnSchemaVersionMinor, }, - MacAddress: epInfo.MacAddress.String(), } + // macAddress type for InfraNIC is like "60:45:bd:12:45:65" + macAddress := epInfo.MacAddress.String() + if epInfo.NICType != cns.InfraNIC { + // convert the format of macAddress that HNS can accept, i.e, "60-45-bd-12-45-65" if NIC type is delegated NIC + macAddress = strings.Join(strings.Split(macAddress, ":"), "-") + } + hcnEndpoint.MacAddress = macAddress + if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.NATInfo); err == nil { for _, epPolicy := range endpointPolicies { hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) @@ -399,6 +408,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( ContainerID: epInfo.ContainerID, PODName: epInfo.PODName, PODNameSpace: epInfo.PODNameSpace, + // SecondaryInterfaces: make(map[string]*InterfaceInfo), } for _, route := range epInfo.Routes { @@ -407,6 +417,19 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( ep.MacAddress, _ = net.ParseMAC(hnsResponse.MacAddress) + // Confirm with TM: when we delete an endpoint, this code is to find ifName from endpoint and then we can delete this endpoint + // ipconfigs := make([]*IPConfig, len(ep.IPAddresses)) + // for i, ipconfig := range ep.IPAddresses { + // ipconfigs[i] = &IPConfig{Address: ipconfig} + // } + + // // Add secondary interfaces info to CNI state file + // ep.SecondaryInterfaces[ep.IfName] = &InterfaceInfo{ + // MacAddress: ep.MacAddress, + // IPConfigs: ipconfigs, + // Routes: ep.Routes, + // } + return ep, nil } diff --git a/network/network_windows.go b/network/network_windows.go index 913bf029b6..6fb92ea923 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/network/hnswrapper" "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" @@ -299,6 +300,11 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *EndpointInfo, extIf *exter return nil, errNetworkModeInvalid } + if nwInfo.NICType == cns.DelegatedVMNIC { + hcnNetwork.Type = hcn.Transparent + hcnNetwork.Flags = hcn.DisableHostPort + } + // Populate subnets. for _, subnet := range nwInfo.Subnets { hnsSubnet := hcn.Subnet{ From 7b380eb90c7605a49ad3d40447c74c429c905785 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 16 Apr 2024 23:05:19 -0700 Subject: [PATCH 053/102] add nic type to windows endpoints --- network/endpoint.go | 4 ++-- network/endpoint_windows.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index c8dc0f167f..2d9591b136 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -153,9 +153,9 @@ type apipaClient interface { } func (epInfo *EndpointInfo) PrettyString() string { - return fmt.Sprintf("Id:%s ContainerID:%s NetNsPath:%s IfName:%s IfIndex:%d MacAddr:%s IPAddrs:%v Gateways:%v Data:%+v", + return fmt.Sprintf("Id:%s ContainerID:%s NetNsPath:%s IfName:%s IfIndex:%d MacAddr:%s IPAddrs:%v Gateways:%v Data:%+v NICType: %s", epInfo.EndpointID, epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName, epInfo.IfIndex, epInfo.MacAddress.String(), epInfo.IPAddresses, - epInfo.Gateways, epInfo.Data) + epInfo.Gateways, epInfo.Data, epInfo.NICType) } func (ifInfo *InterfaceInfo) PrettyString() string { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index cc28a4a56e..5b82016dc6 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -171,6 +171,7 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo, plc platform.ExecC EnableSnatOnHost: epInfo.EnableSnatOnHost, NetNs: epInfo.NetNsPath, ContainerID: epInfo.ContainerID, + NICType: epInfo.NICType, } for _, route := range epInfo.Routes { @@ -409,6 +410,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( PODName: epInfo.PODName, PODNameSpace: epInfo.PODNameSpace, // SecondaryInterfaces: make(map[string]*InterfaceInfo), + NICType: epInfo.NICType, } for _, route := range epInfo.Routes { From 44a56c112fcbd30816468d9ffc15a8da47f5e969 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 18 Apr 2024 16:28:49 -0700 Subject: [PATCH 054/102] move adding an external interface code to run only when creating a new network this change reflects prior behavior, where we would only add an external interface to the statefile if the network (after searching through all external interfaces) was not found currently, if there are multiple interfaces that could be selected as the master, we would add each external interface to the statefile, even if the *network* is associated with one of the existing interfaces while we would still always find the same network (thanks to having a constant NetworkId, regardless of the external interface), you could get an extra empty external interface in your statefile this commit should remove that possibility (the extra external interface shouldn't really matter in the first place though because we always select the external interface that has a matching network created on it) this should be os agnostic --- cni/network/network.go | 17 ++--------------- cni/network/network_windows.go | 2 +- network/endpoint.go | 2 ++ network/network.go | 13 +++++++++++-- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 26543e033a..4ce880c0a4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -680,10 +680,6 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn err := plugin.Errorf("Failed to find the master interface") return nil, err } - logger.Info("Found master interface", zap.String("ifname", masterIfName)) - if err := plugin.addExternalInterface(masterIfName, opt.ifInfo.HostSubnetPrefix.String()); err != nil { - return nil, err - } nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) if err != nil { @@ -787,6 +783,8 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // added the following for delegated vm nic IPAddresses: addresses, MacAddress: opt.ifInfo.MacAddress, + // the following is used for creating an external interface if we can't find an existing network + HostSubnetPrefix: opt.ifInfo.HostSubnetPrefix.String(), } epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) // not specific to delegated or infra @@ -848,17 +846,6 @@ func (plugin *NetPlugin) cleanupAllocationOnError( } } -// Copied from paul's commit -// Add the master as an external interface -func (plugin *NetPlugin) addExternalInterface(masterIfName, hostSubnetPrefix string) error { - err := plugin.nm.AddExternalInterface(masterIfName, hostSubnetPrefix) - if err != nil { - err = plugin.Errorf("Failed to add external interface: %v", err) - return err - } - return nil -} - func (plugin *NetPlugin) getNetworkDNSSettings(nwCfg *cni.NetworkConfig, dns network.DNSInfo) (network.DNSInfo, error) { nwDNSInfo, err := getNetworkDNSSettings(nwCfg, dns) if err != nil { diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 21a3d3e43e..601acc53ab 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -125,7 +125,7 @@ func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *net } } -func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, vethName string) { +func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, _ string) { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { logger.Info("Setting Endpoint Options") var cnetAddressMap []string diff --git a/network/endpoint.go b/network/endpoint.go index 2d9591b136..b423d89a86 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -115,6 +115,8 @@ type EndpointInfo struct { // ServiceCidrs omitted IsIPv6Enabled bool // NICType omitted + + HostSubnetPrefix string // can be used later to add an external interface } // RouteInfo contains information about an IP route. diff --git a/network/network.go b/network/network.go index c77efc019d..44aa5bb199 100644 --- a/network/network.go +++ b/network/network.go @@ -307,12 +307,21 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo for _, epInfo := range epInfos { logger.Info("Creating endpoint and network", zap.String("endpointInfo", epInfo.PrettyString())) - // check if network exists + // check if network exists by searching through all external interfaces for the network _, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkId) if nwGetErr != nil { logger.Info("Existing network not found", zap.String("networkID", epInfo.NetworkId)) + + logger.Info("Found master interface", zap.String("masterIfName", epInfo.MasterIfName)) + + // Add the master as an external interface. + err := nm.AddExternalInterface(epInfo.MasterIfName, epInfo.HostSubnetPrefix) + if err != nil { + return err + } + // Create the network if it is not found - err := nm.CreateNetwork(epInfo) + err = nm.CreateNetwork(epInfo) if err != nil { // TODO: error messages/handling are different in this file // err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) From 4222b83a7ca83f3a59afb728c44b7b7168f28a14 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 19 Apr 2024 17:01:39 -0700 Subject: [PATCH 055/102] update comments, first todo check pass --- cni/network/network.go | 15 ++++++++------- network/endpoint.go | 2 +- network/manager.go | 6 +++--- network/manager_mock.go | 4 +--- network/network.go | 4 ---- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 4ce880c0a4..f1a61731fb 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -561,7 +561,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { var infraSeen bool = false i := 0 for _, ifInfo := range ipamAddResult.interfaceInfo { - // TODO: hopefully I can get natInfo here? + natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) networkID, _ := plugin.getNetworkID(args.Netns, &ifInfo, nwCfg) @@ -603,7 +603,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } defer func() { if err != nil { - // CHECK: can we just keep going through each ep info ignoring if the ep actually exists? + // Delete all endpoints for _, epInfo := range epInfos { deleteErr := plugin.nm.DeleteEndpoint(epInfo.NetworkId, epInfo.EndpointID, epInfo) @@ -692,15 +692,15 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn MasterIfName: masterIfName, AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, BridgeName: opt.ipamAddConfig.nwCfg.Bridge, - EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above - NetworkDNS: nwDNSInfo, // (unused) overridden by endpoint info value POSSIBLE CONFLICT (resolved by making nw and ep dns infos) + EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value; there will be no conflict (confirmed same value as field with same name in epInfo above) + NetworkDNS: nwDNSInfo, // (unused) overridden by endpoint info value; nw and ep dns infos are separated to avoid possible conflicts Policies: opt.policies, // not present in non-infra NetNs: opt.ipamAddConfig.args.Netns, Options: opt.ipamAddConfig.options, DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // not present in non-infra TODO: check if IPV6Mode field can be deprecated // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // not present in non-infra TODO: check if IPV6Mode field can be deprecated // (unused) overridden by endpoint info value; there will be no conflict (onfirmed same value as field with same name in epInfo above) IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, // not present in non-infra - ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value NO CONFLICT: Confirmed same value as field with same name in epInfo above + ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value; there will be no conflict (confirmed same value as field with same name in epInfo above) IsIPv6Enabled: opt.ipv6Enabled, // not present in non-infra NICType: opt.ifInfo.NICType, } @@ -1131,7 +1131,8 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { // populate ep infos here in loop if necessary // delete endpoints for _, epInfo := range epInfos { - // CHECK: networkID or epInfo.networkID (it's not populated when we convert ep to epInfo) + // CHECK: in stateless, network id is not populated in epInfo, but in stateful cni, it is (nw id is used in stateful) + // CHECK: in stateless cni, we do not use the network id, but pass in network id to cover stateful case if err = plugin.nm.DeleteEndpoint(epInfo.NetworkId, epInfo.EndpointID, epInfo); err != nil { // An error will not be returned if the endpoint is not found // return a retriable error so the container runtime will retry this DEL later diff --git a/network/endpoint.go b/network/endpoint.go index b423d89a86..df5462f2df 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -301,7 +301,7 @@ func (ep *endpoint) getInfo() *EndpointInfo { NetworkContainerID: ep.NetworkContainerID, HNSEndpointID: ep.HnsId, HostIfName: ep.HostIfName, - NICType: ep.NICType, // CHECK: Any more fields to convert endpoint into endpoint info? + NICType: ep.NICType, } info.Routes = append(info.Routes, ep.Routes...) diff --git a/network/manager.go b/network/manager.go index 91999a9b3c..137a8f8233 100644 --- a/network/manager.go +++ b/network/manager.go @@ -451,7 +451,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E defer nm.Unlock() if nm.IsStatelessCNIMode() { - // CHECK: Performs same actions as below, but creates the endpoint in a different way (doesn't actually make calls to cns) + // Calls deleteEndpointImpl directly, skipping the get network check; does not call cns return nm.DeleteEndpointState(networkID, epInfo) } @@ -470,7 +470,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *EndpointInfo) error { nw := &network{ - Id: networkID, + Id: networkID, // CHECK: currently unused in stateless cni Mode: opModeTransparentVlan, SnatBridgeIP: "", extIf: &externalInterface{ @@ -512,7 +512,7 @@ func (nm *networkManager) GetEndpointInfo(networkID, endpointID string) (*Endpoi return epInfo, nil } } - // CHECK: what if multiple infra nics or no infra nics? Guess: multi infra nics are interchangeable + return nil, err } diff --git a/network/manager_mock.go b/network/manager_mock.go index 4aacedd823..27adb077e3 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -156,13 +156,11 @@ func (nm *MockNetworkManager) GetNumEndpointsByContainerID(_ string) int { return numEndpoints } -// TODO: understand mock behavior func (nm *MockNetworkManager) SaveState(eps []*endpoint) error { // TODO: Mock behavior for saving the state separate from TestEndpointInfo/NetworkInfo map maybe return nil } -// TODO: understand mock behavior func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*EndpointInfo) error { eps := []*endpoint{} for _, epInfo := range epInfos { @@ -185,8 +183,8 @@ func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*Endp return nm.SaveState(eps) } -// TODO: understand mock behavior func (nm *MockNetworkManager) DeleteState(epInfos []*EndpointInfo) error { + // TODO: Mock behavior for deleting the state separate from TestEndpointInfo/NetworkInfo map maybe return nil } func (nm *MockNetworkManager) GetEndpointInfosFromContainerID(containerID string) []*EndpointInfo { diff --git a/network/network.go b/network/network.go index 44aa5bb199..f6161c0dbd 100644 --- a/network/network.go +++ b/network/network.go @@ -299,8 +299,6 @@ func (nm *networkManager) GetNumEndpointsByContainerID(containerID string) int { return numEndpoints } -// networkID is opt.nwInfo.ID -// cns url is opt.nwCfg.CNSUrl // Creates the network and corresponding endpoint (should be called once during Add) func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*EndpointInfo) error { eps := []*endpoint{} // save endpoints for stateless @@ -323,8 +321,6 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo // Create the network if it is not found err = nm.CreateNetwork(epInfo) if err != nil { - // TODO: error messages/handling are different in this file - // err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) return err } } From 44a4731013bfaee4fbb140f8c581c5aeb1e1d105 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 19 Apr 2024 17:15:57 -0700 Subject: [PATCH 056/102] address some linter issues --- cni/network/invoker.go | 5 +++-- cni/network/invoker_cns.go | 6 +++--- cni/network/network.go | 15 ++++++++------- cns/restserver/ipam.go | 1 - network/endpoint_windows.go | 1 - network/manager.go | 10 ++++++---- network/manager_mock.go | 9 +++++---- network/network_windows.go | 2 +- 8 files changed, 26 insertions(+), 23 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 43cc9a2f2f..907aeca5d7 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -33,8 +33,9 @@ type IPAMAddResult struct { func (ipamAddResult IPAMAddResult) PrettyString() string { pStr := "InterfaceInfo: " - for _, ifInfo := range ipamAddResult.interfaceInfo { - pStr += ifInfo.PrettyString() + for key := range ipamAddResult.interfaceInfo { + val := ipamAddResult.interfaceInfo[key] + pStr += val.PrettyString() } return pStr } diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 4fd3729334..1f6bf4d985 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -412,7 +412,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } ipConfigs := addResult.interfaceInfo[string(info.nicType)].IPConfigs - ipConfigs = append(addResult.interfaceInfo[string(info.nicType)].IPConfigs, + ipConfigs = append(ipConfigs, &network.IPConfig{ Address: net.IPNet{ IP: ip, @@ -428,9 +428,9 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add resRoute := addResult.interfaceInfo[string(info.nicType)].Routes if len(routes) > 0 { - resRoute = append(addResult.interfaceInfo[string(info.nicType)].Routes, routes...) + resRoute = append(resRoute, routes...) } else { // add default routes if none are provided - resRoute = append(addResult.interfaceInfo[string(info.nicType)].Routes, network.RouteInfo{ + resRoute = append(resRoute, network.RouteInfo{ Dst: defaultRouteDstPrefix, Gw: ncgw, }) diff --git a/cni/network/network.go b/cni/network/network.go index f1a61731fb..34f8082c6e 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -402,10 +402,10 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // Add Interfaces to result. // previously just logged the default (infra) interface so this is equivalent behavior cniResult := &cniTypesCurr.Result{} - for _, ifInfo := range ipamAddResult.interfaceInfo { - logger.Info("Exiting add, interface info retrieved", zap.Any("ifInfo", ifInfo)) - if ifInfo.NICType == cns.InfraNIC { - cniResult = convertInterfaceInfoToCniResult(ifInfo, args.IfName) + for key := range ipamAddResult.interfaceInfo { + logger.Info("Exiting add, interface info retrieved", zap.Any("ifInfo", ipamAddResult.interfaceInfo[key])) + if ipamAddResult.interfaceInfo[key].NICType == cns.InfraNIC { + cniResult = convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[key], args.IfName) logger.Info("CNI result generated", zap.Any("cniResult", cniResult)) } } @@ -558,9 +558,10 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { }() epInfos := []*network.EndpointInfo{} - var infraSeen bool = false + infraSeen := false i := 0 - for _, ifInfo := range ipamAddResult.interfaceInfo { + for key := range ipamAddResult.interfaceInfo { + ifInfo := ipamAddResult.interfaceInfo[key] natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS) networkID, _ := plugin.getNetworkID(args.Netns, &ifInfo, nwCfg) @@ -595,7 +596,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // TODO: should this statement be based on the current iteration instead of the constant ifIndex? // TODO figure out where to put telemetry: sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", // ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) - i += 1 + i++ } cnsclient, err := cnscli.New(nwCfg.CNSUrl, defaultRequestTimeout) if err != nil { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 420bbb13f4..1031b15c3c 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -1158,7 +1158,6 @@ func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req map[ return nil } return errors.New("[updateEndpoint] endpoint could not be found in the statefile") - } // verifyUpdateEndpointStateRequest verify the CNI request body for the UpdateENdpointState API diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 5b82016dc6..54ca6bfc4d 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -76,7 +76,6 @@ func (nw *network) newEndpointImpl( _ ipTablesClient, epInfo *EndpointInfo, ) (*endpoint, error) { - if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { if err != nil { return nil, err diff --git a/network/manager.go b/network/manager.go index 137a8f8233..e63799d595 100644 --- a/network/manager.go +++ b/network/manager.go @@ -341,17 +341,17 @@ func (nm *networkManager) DeleteNetwork(networkID string) error { } // GetNetworkInfo returns information about the given network. -func (nm *networkManager) GetNetworkInfo(networkId string) (EndpointInfo, error) { +func (nm *networkManager) GetNetworkInfo(networkID string) (EndpointInfo, error) { nm.Lock() defer nm.Unlock() - nw, err := nm.getNetwork(networkId) + nw, err := nm.getNetwork(networkID) if err != nil { return EndpointInfo{}, err } nwInfo := EndpointInfo{ - NetworkId: networkId, + NetworkId: networkID, Subnets: nw.Subnets, Mode: nw.Mode, EnableSnatOnHost: nw.EnableSnatOnHost, @@ -708,7 +708,8 @@ func (nm *networkManager) SaveState(eps []*endpoint) error { // once endpoints and networks are in-memory, save once return nm.save() } -func (nm *networkManager) DeleteState(epInfos []*EndpointInfo) error { + +func (nm *networkManager) DeleteState(_ []*EndpointInfo) error { nm.Lock() defer nm.Unlock() @@ -749,6 +750,7 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI } return ret } + func (nm *networkManager) GetEndpointInfosFromContainerID(containerID string) []*EndpointInfo { ret := []*EndpointInfo{} for _, extIf := range nm.ExternalInterfaces { diff --git a/network/manager_mock.go b/network/manager_mock.go index 27adb077e3..c2344feefb 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -56,7 +56,6 @@ func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (EndpointInfo, er // CreateEndpoint mock // TODO: Fix mock behavior because create endpoint no longer also saves the state func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfo *EndpointInfo) (*endpoint, error) { - if err := nm.TestEndpointClient.AddEndpoints(epInfo); err != nil { return nil, err } @@ -156,7 +155,7 @@ func (nm *MockNetworkManager) GetNumEndpointsByContainerID(_ string) int { return numEndpoints } -func (nm *MockNetworkManager) SaveState(eps []*endpoint) error { +func (nm *MockNetworkManager) SaveState(_ []*endpoint) error { // TODO: Mock behavior for saving the state separate from TestEndpointInfo/NetworkInfo map maybe return nil } @@ -183,10 +182,11 @@ func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*Endp return nm.SaveState(eps) } -func (nm *MockNetworkManager) DeleteState(epInfos []*EndpointInfo) error { +func (nm *MockNetworkManager) DeleteState(_ []*EndpointInfo) error { // TODO: Mock behavior for deleting the state separate from TestEndpointInfo/NetworkInfo map maybe return nil } + func (nm *MockNetworkManager) GetEndpointInfosFromContainerID(containerID string) []*EndpointInfo { ret := []*EndpointInfo{} for _, epInfo := range nm.TestEndpointInfoMap { @@ -196,6 +196,7 @@ func (nm *MockNetworkManager) GetEndpointInfosFromContainerID(containerID string } return ret } -func (nm *MockNetworkManager) GetEndpointState(networkID, containerID string) ([]*EndpointInfo, error) { + +func (nm *MockNetworkManager) GetEndpointState(_, _ string) ([]*EndpointInfo, error) { return []*EndpointInfo{}, nil } diff --git a/network/network_windows.go b/network/network_windows.go index 6fb92ea923..b5e9eabaad 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -469,5 +469,5 @@ func (nm *networkManager) deleteNetworkImplHnsV2(nw *network) error { return err } -func getNetworkInfoImpl(nwInfo *EndpointInfo, nw *network) { +func getNetworkInfoImpl(_ *EndpointInfo, _ *network) { } From b2c1ffea71b8685eeddf20caae76ef788f4fb0ce Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 19 Apr 2024 17:44:38 -0700 Subject: [PATCH 057/102] rename networkId to networkID in endpoint info ran package tests in windows and linux for cni and network packages ran package tests in linux for cns restserver all have expected outputs (either pass, or also fails on master branch) --- cni/network/network.go | 10 +++++----- cnm/network/network.go | 2 +- network/endpoint.go | 2 +- network/manager.go | 4 ++-- network/manager_mock.go | 8 ++++---- network/network.go | 14 +++++++------- network/network_linux.go | 2 +- network/network_test.go | 8 ++++---- network/network_windows.go | 8 ++++---- network/network_windows_test.go | 16 ++++++++-------- 10 files changed, 37 insertions(+), 37 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 34f8082c6e..40bff721e8 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -607,7 +607,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // Delete all endpoints for _, epInfo := range epInfos { - deleteErr := plugin.nm.DeleteEndpoint(epInfo.NetworkId, epInfo.EndpointID, epInfo) + deleteErr := plugin.nm.DeleteEndpoint(epInfo.NetworkID, epInfo.EndpointID, epInfo) if deleteErr != nil { // we already do not return an error when the endpoint is not found, so deleteErr is a real error logger.Error("Could not delete endpoint after detecting add failure", zap.String("epInfo", epInfo.PrettyString()), zap.Error(deleteErr)) @@ -688,7 +688,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn } // this struct is for organization and represents fields from the network to be merged into the endpoint info later nwInfo = network.EndpointInfo{ - NetworkId: opt.networkID, + NetworkID: opt.networkID, Mode: opt.ipamAddConfig.nwCfg.Mode, MasterIfName: masterIfName, AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, @@ -738,7 +738,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. // IT will result in unpredictable behavior if API server decides to // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", nwInfo.NetworkId, opt.args.ContainerID, opt.args.IfName) + vethName = fmt.Sprintf("%s%s%s", nwInfo.NetworkID, opt.args.ContainerID, opt.args.IfName) } // for secondary (Populate addresses) @@ -813,7 +813,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // populate endpoint info with network info fields epInfo.MasterIfName = nwInfo.MasterIfName epInfo.AdapterName = nwInfo.AdapterName - epInfo.NetworkId = nwInfo.NetworkId + epInfo.NetworkID = nwInfo.NetworkID epInfo.Mode = nwInfo.Mode epInfo.Subnets = nwInfo.Subnets epInfo.PodSubnet = nwInfo.PodSubnet @@ -1134,7 +1134,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { for _, epInfo := range epInfos { // CHECK: in stateless, network id is not populated in epInfo, but in stateful cni, it is (nw id is used in stateful) // CHECK: in stateless cni, we do not use the network id, but pass in network id to cover stateful case - if err = plugin.nm.DeleteEndpoint(epInfo.NetworkId, epInfo.EndpointID, epInfo); err != nil { + if err = plugin.nm.DeleteEndpoint(epInfo.NetworkID, epInfo.EndpointID, epInfo); err != nil { // An error will not be returned if the endpoint is not found // return a retriable error so the container runtime will retry this DEL later // the implementation of this function returns nil if the endpoint doens't exist, so diff --git a/cnm/network/network.go b/cnm/network/network.go index 5d7d352d37..733dbad85e 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -146,7 +146,7 @@ func (plugin *netPlugin) createNetwork(w http.ResponseWriter, r *http.Request) { // Process request. nwInfo := network.EndpointInfo{ - NetworkId: req.NetworkID, + NetworkID: req.NetworkID, Options: req.Options, } diff --git a/network/endpoint.go b/network/endpoint.go index df5462f2df..eca4667b61 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -99,7 +99,7 @@ type EndpointInfo struct { MasterIfName string // related to HostIfName? AdapterName string - NetworkId string // referred to as Id in NetworkInfo + NetworkID string // referred to as Id in NetworkInfo Mode string Subnets []SubnetInfo PodSubnet SubnetInfo diff --git a/network/manager.go b/network/manager.go index e63799d595..cb91fdd3a3 100644 --- a/network/manager.go +++ b/network/manager.go @@ -351,7 +351,7 @@ func (nm *networkManager) GetNetworkInfo(networkID string) (EndpointInfo, error) } nwInfo := EndpointInfo{ - NetworkId: networkID, + NetworkID: networkID, Subnets: nw.Subnets, Mode: nw.Mode, EnableSnatOnHost: nw.EnableSnatOnHost, @@ -758,7 +758,7 @@ func (nm *networkManager) GetEndpointInfosFromContainerID(containerID string) [] for _, ep := range nw.Endpoints { if ep.ContainerID == containerID { val := ep.getInfo() - val.NetworkId = networkID // endpoint doesn't contain the network id + val.NetworkID = networkID // endpoint doesn't contain the network id ret = append(ret, val) } } diff --git a/network/manager_mock.go b/network/manager_mock.go index c2344feefb..224213be75 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -36,7 +36,7 @@ func (nm *MockNetworkManager) AddExternalInterface(ifName string, subnet string) // CreateNetwork mock func (nm *MockNetworkManager) CreateNetwork(nwInfo *EndpointInfo) error { - nm.TestNetworkInfoMap[nwInfo.NetworkId] = nwInfo + nm.TestNetworkInfoMap[nwInfo.NetworkID] = nwInfo return nil } @@ -147,7 +147,7 @@ func (nm *MockNetworkManager) GetNumEndpointsByContainerID(_ string) int { numEndpoints := 0 for _, network := range nm.TestNetworkInfoMap { - if _, err := nm.GetAllEndpoints(network.NetworkId); err == nil { + if _, err := nm.GetAllEndpoints(network.NetworkID); err == nil { numEndpoints++ } } @@ -163,7 +163,7 @@ func (nm *MockNetworkManager) SaveState(_ []*endpoint) error { func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*EndpointInfo) error { eps := []*endpoint{} for _, epInfo := range epInfos { - _, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkId) + _, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkID) if nwGetErr != nil { err := nm.CreateNetwork(epInfo) if err != nil { @@ -171,7 +171,7 @@ func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*Endp } } - ep, err := nm.CreateEndpoint(client, epInfo.NetworkId, epInfo) + ep, err := nm.CreateEndpoint(client, epInfo.NetworkID, epInfo) if err != nil { return errors.Wrap(err, "Failed to create endpoint") } diff --git a/network/network.go b/network/network.go index f6161c0dbd..4d21670c71 100644 --- a/network/network.go +++ b/network/network.go @@ -169,7 +169,7 @@ func (nm *networkManager) newNetwork(epInfo *EndpointInfo) (*network, error) { logger.Info("Creating", zap.String("network", epInfo.PrettyString())) defer func() { if err != nil { - logger.Error("Failed to create network", zap.String("id", epInfo.NetworkId), zap.Error(err)) + logger.Error("Failed to create network", zap.String("id", epInfo.NetworkID), zap.Error(err)) } }() @@ -192,7 +192,7 @@ func (nm *networkManager) newNetwork(epInfo *EndpointInfo) (*network, error) { } // Make sure this network does not already exist. - if extIf.Networks[epInfo.NetworkId] != nil { + if extIf.Networks[epInfo.NetworkID] != nil { err = errNetworkExists return nil, err } @@ -205,9 +205,9 @@ func (nm *networkManager) newNetwork(epInfo *EndpointInfo) (*network, error) { // Add the network object. nw.Subnets = epInfo.Subnets - extIf.Networks[epInfo.NetworkId] = nw + extIf.Networks[epInfo.NetworkID] = nw - logger.Info("Created network on interface", zap.String("id", epInfo.NetworkId), zap.String("Name", extIf.Name)) + logger.Info("Created network on interface", zap.String("id", epInfo.NetworkID), zap.String("Name", extIf.Name)) return nw, nil } @@ -306,9 +306,9 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo for _, epInfo := range epInfos { logger.Info("Creating endpoint and network", zap.String("endpointInfo", epInfo.PrettyString())) // check if network exists by searching through all external interfaces for the network - _, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkId) + _, nwGetErr := nm.GetNetworkInfo(epInfo.NetworkID) if nwGetErr != nil { - logger.Info("Existing network not found", zap.String("networkID", epInfo.NetworkId)) + logger.Info("Existing network not found", zap.String("networkID", epInfo.NetworkID)) logger.Info("Found master interface", zap.String("masterIfName", epInfo.MasterIfName)) @@ -325,7 +325,7 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo } } - ep, err := nm.CreateEndpoint(cnsclient, epInfo.NetworkId, epInfo) + ep, err := nm.CreateEndpoint(cnsclient, epInfo.NetworkID, epInfo) if err != nil { return errors.Wrap(err, "Failed to create endpoint") } diff --git a/network/network_linux.go b/network/network_linux.go index 476f8fb807..c2a0bcc012 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -116,7 +116,7 @@ func (nm *networkManager) newNetworkImpl(epInfo *EndpointInfo, extIf *externalIn // Create the network object. nw := &network{ - Id: epInfo.NetworkId, + Id: epInfo.NetworkID, Mode: epInfo.Mode, Endpoints: make(map[string]*endpoint), extIf: extIf, diff --git a/network/network_test.go b/network/network_test.go index 9da8b20788..2f48451022 100644 --- a/network/network_test.go +++ b/network/network_test.go @@ -151,7 +151,7 @@ var _ = Describe("Test Network", func() { } nm.ExternalInterfaces["eth0"].Networks["nw"] = &network{} nwInfo := &EndpointInfo{ - NetworkId: "nw", + NetworkID: "nw", MasterIfName: "eth0", } nw, err := nm.newNetwork(nwInfo) @@ -170,7 +170,7 @@ var _ = Describe("Test Network", func() { Networks: map[string]*network{}, } nwInfo := &EndpointInfo{ - NetworkId: "nw", + NetworkID: "nw", MasterIfName: "eth0", Mode: opModeTransparent, IPV6Mode: IPV6Nat, @@ -178,7 +178,7 @@ var _ = Describe("Test Network", func() { nw, err := nm.newNetwork(nwInfo) Expect(err).To(BeNil()) Expect(nw).NotTo(BeNil()) - Expect(nw.Id).To(Equal(nwInfo.NetworkId)) + Expect(nw.Id).To(Equal(nwInfo.NetworkID)) }) }) @@ -192,7 +192,7 @@ var _ = Describe("Test Network", func() { Networks: map[string]*network{}, } nwInfo := &EndpointInfo{ - NetworkId: "nw", + NetworkID: "nw", MasterIfName: "eth0", Mode: opModeTransparentVlan, } diff --git a/network/network_windows.go b/network/network_windows.go index b5e9eabaad..fbfcc5d142 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -112,7 +112,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *EndpointInfo, extIf *exter // Initialize HNS network. hnsNetwork := &hcsshim.HNSNetwork{ - Name: nwInfo.NetworkId, + Name: nwInfo.NetworkID, NetworkAdapterName: networkAdapterName, DNSServerList: strings.Join(nwInfo.NetworkDNS.Servers, ","), Policies: policy.SerializePolicies(policy.NetworkPolicy, nwInfo.Policies, nil, false, false), @@ -173,7 +173,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *EndpointInfo, extIf *exter // Create the network object. nw := &network{ - Id: nwInfo.NetworkId, + Id: nwInfo.NetworkID, HnsId: hnsResponse.Id, Mode: nwInfo.Mode, Endpoints: make(map[string]*endpoint), @@ -231,7 +231,7 @@ func (nm *networkManager) appIPV6RouteEntry(nwInfo *EndpointInfo) error { func (nm *networkManager) configureHcnNetwork(nwInfo *EndpointInfo, extIf *externalInterface) (*hcn.HostComputeNetwork, error) { // Initialize HNS network. hcnNetwork := &hcn.HostComputeNetwork{ - Name: nwInfo.NetworkId, + Name: nwInfo.NetworkID, Dns: hcn.Dns{ Domain: nwInfo.NetworkDNS.Suffix, ServerList: nwInfo.NetworkDNS.Servers, @@ -406,7 +406,7 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *EndpointInfo, extIf *exter // Create the network object. nw := &network{ - Id: nwInfo.NetworkId, + Id: nwInfo.NetworkID, HnsId: hnsResponse.Id, Mode: nwInfo.Mode, Endpoints: make(map[string]*endpoint), diff --git a/network/network_windows_test.go b/network/network_windows_test.go index 49a9488e43..e9990ce931 100644 --- a/network/network_windows_test.go +++ b/network/network_windows_test.go @@ -35,7 +35,7 @@ func TestNewAndDeleteNetworkImplHnsV2(t *testing.T) { Hnsv2 = hnswrapper.NewHnsv2wrapperFake() nwInfo := &EndpointInfo{ - NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + NetworkID: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -76,7 +76,7 @@ func TestSuccesfulNetworkCreationWhenAlreadyExists(t *testing.T) { // network name is derived from network info id nwInfo := &EndpointInfo{ - NetworkId: "azure-vlan1-172-28-1-0_24", + NetworkID: "azure-vlan1-172-28-1-0_24", MasterIfName: "eth0", Mode: "bridge", } @@ -109,7 +109,7 @@ func TestNewNetworkImplHnsV2WithTimeout(t *testing.T) { } nwInfo := &EndpointInfo{ - NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + NetworkID: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -132,7 +132,7 @@ func TestDeleteNetworkImplHnsV2WithTimeout(t *testing.T) { } nwInfo := &EndpointInfo{ - NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + NetworkID: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -181,7 +181,7 @@ func TestNewNetworkImplHnsV1WithTimeout(t *testing.T) { } nwInfo := &EndpointInfo{ - NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + NetworkID: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -204,7 +204,7 @@ func TestDeleteNetworkImplHnsV1WithTimeout(t *testing.T) { } nwInfo := &EndpointInfo{ - NetworkId: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", + NetworkID: "d3e97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", } @@ -261,7 +261,7 @@ func TestAddIPv6DefaultRoute(t *testing.T) { } nwInfo := &EndpointInfo{ - NetworkId: "d3f97a83-ba4c-45d5-ba88-dc56757ece28", + NetworkID: "d3f97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", Subnets: networkSubnetInfo, @@ -304,7 +304,7 @@ func TestFailToAddIPv6DefaultRoute(t *testing.T) { } nwInfo := &EndpointInfo{ - NetworkId: "d3f97a83-ba4c-45d5-ba88-dc56757ece28", + NetworkID: "d3f97a83-ba4c-45d5-ba88-dc56757ece28", MasterIfName: "eth0", Mode: "bridge", Subnets: networkSubnetInfo, From fc163244f2b79bf3a6239d43fc3a6550632e3c97 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 19 Apr 2024 18:15:52 -0700 Subject: [PATCH 058/102] address linter issues --- cni/network/invoker_azure.go | 2 +- cni/network/multitenancy.go | 3 ++- cni/network/network.go | 26 ++++++++++++++------------ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index 016bf6601b..91057ff887 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -114,7 +114,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er } // TODO: changed how host subnet prefix populated (check) - var hostSubnetPrefix = net.IPNet{} + hostSubnetPrefix := net.IPNet{} if len(result.IPs) > 0 { hostSubnetPrefix = result.IPs[0].Address } diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 39b757bca3..6481d241f7 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "os" + "strconv" "strings" "time" @@ -231,7 +232,7 @@ func (m *Multitenancy) GetAllNetworkContainers( ifInfo.NICType = cns.InfraNIC // assuming we only assign infra nics in this function - ipamResult.interfaceInfo[string(ifInfo.NICType)+fmt.Sprint(i)] = ifInfo + ipamResult.interfaceInfo[string(ifInfo.NICType)+strconv.Itoa(i)] = ifInfo } return ipamResult, err diff --git a/cni/network/network.go b/cni/network/network.go index 40bff721e8..829c6c7d26 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -640,6 +640,8 @@ func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string { return plugin.findMasterInterfaceBySubnet(opt.ipamAddConfig.nwCfg, &opt.ifInfo.HostSubnetPrefix) case cns.DelegatedVMNIC: return plugin.findInterfaceByMAC(opt.ifInfo.MacAddress.String()) + case cns.BackendNIC: + return "" default: return "" } @@ -693,20 +695,20 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn MasterIfName: masterIfName, AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, BridgeName: opt.ipamAddConfig.nwCfg.Bridge, - EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) overridden by endpoint info value; there will be no conflict (confirmed same value as field with same name in epInfo above) - NetworkDNS: nwDNSInfo, // (unused) overridden by endpoint info value; nw and ep dns infos are separated to avoid possible conflicts - Policies: opt.policies, // not present in non-infra + EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) same value as field with same name in epInfo above + NetworkDNS: nwDNSInfo, // (unused) nw and ep dns infos are separated to avoid possible conflicts + Policies: opt.policies, // present infra only NetNs: opt.ipamAddConfig.args.Netns, Options: opt.ipamAddConfig.options, DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // not present in non-infra TODO: check if IPV6Mode field can be deprecated // (unused) overridden by endpoint info value; there will be no conflict (onfirmed same value as field with same name in epInfo above) - IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, // not present in non-infra - ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) overridden by endpoint info value; there will be no conflict (confirmed same value as field with same name in epInfo above) - IsIPv6Enabled: opt.ipv6Enabled, // not present in non-infra + IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // present infra only TODO: check if IPV6Mode field can be deprecated (unused) + IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, // present infra only + ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) same value as field with same name in epInfo above + IsIPv6Enabled: opt.ipv6Enabled, // present infra only NICType: opt.ifInfo.NICType, } - if err := addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { + if err = addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) return nil, err } @@ -742,10 +744,10 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn } // for secondary (Populate addresses) - // intially only for infra nic but now applied to all nic types - var addresses []net.IPNet - for _, ipconfig := range opt.ifInfo.IPConfigs { - addresses = append(addresses, ipconfig.Address) + // initially only for infra nic but now applied to all nic types + var addresses = make([]net.IPNet, len(opt.ifInfo.IPConfigs)) + for i, ipconfig := range opt.ifInfo.IPConfigs { + addresses[i] = ipconfig.Address } // generate endpoint info From 0f004925cfb37a4594df63a4259242c37cf07785 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Sun, 21 Apr 2024 22:00:09 -0700 Subject: [PATCH 059/102] preserve more logs and reduce timeout for restart for debugging --- .pipelines/cni/singletenancy/cniv2-template.yaml | 2 +- cni/log/logger.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/cni/singletenancy/cniv2-template.yaml b/.pipelines/cni/singletenancy/cniv2-template.yaml index 261b44f98c..27475670fc 100644 --- a/.pipelines/cni/singletenancy/cniv2-template.yaml +++ b/.pipelines/cni/singletenancy/cniv2-template.yaml @@ -197,7 +197,7 @@ stages: condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'restartNode') , contains(variables.CONTROL_SCENARIO, 'all') ) ) displayName: "Restart Test" dependsOn: deploy_pods - timeoutInMinutes: 90 # Windows podsubnet takes an extended amount of time to reconcile + timeoutInMinutes: 20 # Windows podsubnet takes an extended amount of time to reconcile steps: - template: ../load-test-templates/restart-node-template.yaml parameters: diff --git a/cni/log/logger.go b/cni/log/logger.go index 6ee40b6e1d..0d6540e15a 100644 --- a/cni/log/logger.go +++ b/cni/log/logger.go @@ -16,7 +16,7 @@ var ( const ( maxLogFileSizeInMb = 5 - maxLogFileCount = 8 + maxLogFileCount = 110 ) func initZapLog(logFile string) *zap.Logger { From 2e35953ccd1ad5db3def865099251dbaa0fbf4be Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 23 Apr 2024 11:37:33 -0700 Subject: [PATCH 060/102] clean comments and rename for clarity if we use the endpoint info for the network info fields, we name it nwInfo as a hint --- network/endpoint.go | 34 +++++++++++-------------- network/network.go | 26 +++++++++---------- network/network_linux.go | 54 ++++++++++++++++++++-------------------- 3 files changed, 54 insertions(+), 60 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index eca4667b61..4428b82d27 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -52,7 +52,7 @@ type endpoint struct { PODName string `json:",omitempty"` PODNameSpace string `json:",omitempty"` InfraVnetAddressSpace string `json:",omitempty"` - NetNs string `json:",omitempty"` + NetNs string `json:",omitempty"` // used in windows // SecondaryInterfaces is a map of interface name to InterfaceInfo SecondaryInterfaces map[string]*InterfaceInfo // Store nic type since we no longer populate SecondaryInterfaces @@ -95,26 +95,20 @@ type EndpointInfo struct { NICType cns.NICType SkipDefaultRoutes bool HNSEndpointID string - HostIfName string - - MasterIfName string // related to HostIfName? - AdapterName string - NetworkID string // referred to as Id in NetworkInfo - Mode string - Subnets []SubnetInfo - PodSubnet SubnetInfo - // DNS omitted - // Policies omitted - BridgeName string - // EnableSnatOnHost omitted - NetNs string // related to NetNsPath? - Options map[string]interface{} // related to Data? + HostIfName string // unused in windows, and in linux + // Fields related to the network are below + MasterIfName string + AdapterName string + NetworkID string + Mode string + Subnets []SubnetInfo + PodSubnet SubnetInfo + BridgeName string + NetNs string // used in windows + Options map[string]interface{} DisableHairpinOnHostInterface bool - // IPV6Mode omitted - IPAMType string - // ServiceCidrs omitted - IsIPv6Enabled bool - // NICType omitted + IPAMType string + IsIPv6Enabled bool HostSubnetPrefix string // can be used later to add an external interface } diff --git a/network/network.go b/network/network.go index 4d21670c71..9e0af3c681 100644 --- a/network/network.go +++ b/network/network.go @@ -162,29 +162,29 @@ func (nm *networkManager) findExternalInterfaceByName(ifName string) *externalIn } // NewNetwork creates a new container network. -func (nm *networkManager) newNetwork(epInfo *EndpointInfo) (*network, error) { +func (nm *networkManager) newNetwork(nwInfo *EndpointInfo) (*network, error) { var nw *network var err error - logger.Info("Creating", zap.String("network", epInfo.PrettyString())) + logger.Info("Creating", zap.String("network", nwInfo.PrettyString())) defer func() { if err != nil { - logger.Error("Failed to create network", zap.String("id", epInfo.NetworkID), zap.Error(err)) + logger.Error("Failed to create network", zap.String("id", nwInfo.NetworkID), zap.Error(err)) } }() // Set defaults. - if epInfo.Mode == "" { - epInfo.Mode = opModeDefault + if nwInfo.Mode == "" { + nwInfo.Mode = opModeDefault } // If the master interface name is provided, find the external interface by name // else use subnet to to find the interface var extIf *externalInterface - if len(strings.TrimSpace(epInfo.MasterIfName)) > 0 { - extIf = nm.findExternalInterfaceByName(epInfo.MasterIfName) + if len(strings.TrimSpace(nwInfo.MasterIfName)) > 0 { + extIf = nm.findExternalInterfaceByName(nwInfo.MasterIfName) } else { - extIf = nm.findExternalInterfaceBySubnet(epInfo.Subnets[0].Prefix.String()) + extIf = nm.findExternalInterfaceBySubnet(nwInfo.Subnets[0].Prefix.String()) } if extIf == nil { err = errSubnetNotFound @@ -192,22 +192,22 @@ func (nm *networkManager) newNetwork(epInfo *EndpointInfo) (*network, error) { } // Make sure this network does not already exist. - if extIf.Networks[epInfo.NetworkID] != nil { + if extIf.Networks[nwInfo.NetworkID] != nil { err = errNetworkExists return nil, err } // Call the OS-specific implementation. - nw, err = nm.newNetworkImpl(epInfo, extIf) + nw, err = nm.newNetworkImpl(nwInfo, extIf) if err != nil { return nil, err } // Add the network object. - nw.Subnets = epInfo.Subnets - extIf.Networks[epInfo.NetworkID] = nw + nw.Subnets = nwInfo.Subnets + extIf.Networks[nwInfo.NetworkID] = nw - logger.Info("Created network on interface", zap.String("id", epInfo.NetworkID), zap.String("Name", extIf.Name)) + logger.Info("Created network on interface", zap.String("id", nwInfo.NetworkID), zap.String("Name", extIf.Name)) return nw, nil } diff --git a/network/network_linux.go b/network/network_linux.go index c2a0bcc012..a8019de06e 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -59,22 +59,22 @@ func newErrorNetworkManager(errStr string) error { type route netlink.Route // NewNetworkImpl creates a new container network. -func (nm *networkManager) newNetworkImpl(epInfo *EndpointInfo, extIf *externalInterface) (*network, error) { +func (nm *networkManager) newNetworkImpl(nwInfo *EndpointInfo, extIf *externalInterface) (*network, error) { // Connect the external interface. var ( vlanid int ifName string ) - opt, _ := epInfo.Options[genericData].(map[string]interface{}) - logger.Info("opt options", zap.Any("opt", opt), zap.Any("options", epInfo.Options)) + opt, _ := nwInfo.Options[genericData].(map[string]interface{}) + logger.Info("opt options", zap.Any("opt", opt), zap.Any("options", nwInfo.Options)) - switch epInfo.Mode { + switch nwInfo.Mode { case opModeTunnel: fallthrough case opModeBridge: logger.Info("create bridge") ifName = extIf.BridgeName - if err := nm.connectExternalInterface(extIf, epInfo); err != nil { + if err := nm.connectExternalInterface(extIf, nwInfo); err != nil { return nil, err } @@ -84,7 +84,7 @@ func (nm *networkManager) newNetworkImpl(epInfo *EndpointInfo, extIf *externalIn case opModeTransparent: logger.Info("Transparent mode") ifName = extIf.Name - if epInfo.IPV6Mode != "" { + if nwInfo.IPV6Mode != "" { nu := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) if err := nu.EnableIPV6Forwarding(); err != nil { return nil, fmt.Errorf("Ipv6 forwarding failed: %w", err) @@ -108,7 +108,7 @@ func (nm *networkManager) newNetworkImpl(epInfo *EndpointInfo, extIf *externalIn return nil, errNetworkModeInvalid } - err := nm.handleCommonOptions(ifName, epInfo) + err := nm.handleCommonOptions(ifName, nwInfo) if err != nil { logger.Error("handleCommonOptions failed with", zap.Error(err)) return nil, err @@ -116,28 +116,28 @@ func (nm *networkManager) newNetworkImpl(epInfo *EndpointInfo, extIf *externalIn // Create the network object. nw := &network{ - Id: epInfo.NetworkID, - Mode: epInfo.Mode, + Id: nwInfo.NetworkID, + Mode: nwInfo.Mode, Endpoints: make(map[string]*endpoint), extIf: extIf, VlanId: vlanid, - DNS: epInfo.NetworkDNS, - EnableSnatOnHost: epInfo.EnableSnatOnHost, + DNS: nwInfo.NetworkDNS, + EnableSnatOnHost: nwInfo.EnableSnatOnHost, } return nw, nil } -func (nm *networkManager) handleCommonOptions(ifName string, epInfo *EndpointInfo) error { +func (nm *networkManager) handleCommonOptions(ifName string, nwInfo *EndpointInfo) error { var err error - if routes, exists := epInfo.Options[RoutesKey]; exists { + if routes, exists := nwInfo.Options[RoutesKey]; exists { err = addRoutes(nm.netlink, nm.netio, ifName, routes.([]RouteInfo)) if err != nil { return err } } - if iptcmds, exists := epInfo.Options[IPTablesKey]; exists { + if iptcmds, exists := nwInfo.Options[IPTablesKey]; exists { err = nm.addToIptables(iptcmds.([]iptables.IPTableEntry)) if err != nil { return err @@ -460,7 +460,7 @@ func (nm *networkManager) applyDNSConfig(extIf *externalInterface, ifName string } // ConnectExternalInterface connects the given host interface to a bridge. -func (nm *networkManager) connectExternalInterface(extIf *externalInterface, epInfo *EndpointInfo) error { +func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *EndpointInfo) error { var ( err error networkClient NetworkClient @@ -482,16 +482,16 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, epI } // If a bridge name is not specified, generate one based on the external interface index. - bridgeName := epInfo.BridgeName + bridgeName := nwInfo.BridgeName if bridgeName == "" { bridgeName = fmt.Sprintf("%s%d", bridgePrefix, hostIf.Index) } - opt, _ := epInfo.Options[genericData].(map[string]interface{}) + opt, _ := nwInfo.Options[genericData].(map[string]interface{}) if opt != nil && opt[VlanIDKey] != nil { networkClient = NewOVSClient(bridgeName, extIf.Name, ovsctl.NewOvsctl(), nm.netlink, nm.plClient) } else { - networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *epInfo, nm.netlink, nm.plClient) + networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *nwInfo, nm.netlink, nm.plClient) } // Check if the bridge already exists. @@ -576,7 +576,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, epI } // External interface hairpin on. - if !epInfo.DisableHairpinOnHostInterface { + if !nwInfo.DisableHairpinOnHostInterface { logger.Info("Setting link hairpin on", zap.String("Name", hostIf.Name)) if err = networkClient.SetHairpinOnHostInterface(true); err != nil { return err @@ -599,14 +599,14 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, epI logger.Info("Applied dns config on", zap.Any("DNSInfo", extIf.DNSInfo), zap.String("bridgeName", bridgeName)) } - if epInfo.IPV6Mode == IPV6Nat { + if nwInfo.IPV6Mode == IPV6Nat { // adds pod cidr gateway ip to bridge - if err = nm.addIpv6NatGateway(epInfo); err != nil { + if err = nm.addIpv6NatGateway(nwInfo); err != nil { logger.Error("Adding IPv6 Nat Gateway failed with", zap.Error(err)) return err } - if err = nm.addIpv6SnatRule(extIf, epInfo); err != nil { + if err = nm.addIpv6SnatRule(extIf, nwInfo); err != nil { logger.Error("Adding IPv6 Snat Rule failed with", zap.Error(err)) return err } @@ -664,16 +664,16 @@ func (nm *networkManager) addToIptables(cmds []iptables.IPTableEntry) error { } // Add ipv6 nat gateway IP on bridge -func (nm *networkManager) addIpv6NatGateway(epInfo *EndpointInfo) error { +func (nm *networkManager) addIpv6NatGateway(nwInfo *EndpointInfo) error { logger.Info("Adding ipv6 nat gateway on azure bridge") - for _, subnetInfo := range epInfo.Subnets { + for _, subnetInfo := range nwInfo.Subnets { if subnetInfo.Family == platform.AfINET6 { ipAddr := []net.IPNet{{ IP: subnetInfo.Gateway, Mask: subnetInfo.Prefix.Mask, }} nuc := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) - err := nuc.AssignIPToInterface(epInfo.BridgeName, ipAddr) + err := nuc.AssignIPToInterface(nwInfo.BridgeName, ipAddr) if err != nil { return newErrorNetworkManager(err.Error()) } @@ -684,13 +684,13 @@ func (nm *networkManager) addIpv6NatGateway(epInfo *EndpointInfo) error { } // snat ipv6 traffic to secondary ipv6 ip before leaving VM -func (nm *networkManager) addIpv6SnatRule(extIf *externalInterface, epInfo *EndpointInfo) error { +func (nm *networkManager) addIpv6SnatRule(extIf *externalInterface, nwInfo *EndpointInfo) error { var ( ipv6SnatRuleSet bool ipv6SubnetPrefix net.IPNet ) - for _, subnet := range epInfo.Subnets { + for _, subnet := range nwInfo.Subnets { if subnet.Family == platform.AfINET6 { ipv6SubnetPrefix = subnet.Prefix break From 91662933448866eab4eec2d3b844cb4dc2d0dbe8 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 23 Apr 2024 18:14:47 -0700 Subject: [PATCH 061/102] address more linter issues linux network, restserver, and cni package tests pass --- cni/network/invoker_cns_test.go | 1 - cni/network/multitenancy_mock.go | 4 ++-- cni/network/network.go | 7 +++---- cni/network/network_test.go | 6 +++--- cni/network/network_windows.go | 5 +++-- network/endpoint_windows.go | 14 ++------------ network/manager_mock.go | 3 +-- network/network.go | 3 +-- 8 files changed, 15 insertions(+), 28 deletions(-) diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index a6858c6b0e..b6f5d48730 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -500,7 +500,6 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { } } if ifInfo.NICType == cns.InfraNIC { - t.Logf("%s", &ifInfo.HostSubnetPrefix) // TODO: Remove later require.Equalf(tt.wantDefaultResult, ifInfo, "incorrect default response") } } diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index 9a34d5a569..de6c7b8eb5 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -3,9 +3,9 @@ package network import ( "context" "errors" - "fmt" "net" "runtime" + "strconv" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" @@ -171,7 +171,7 @@ func (m *MockMultitenancy) GetAllNetworkContainers( ifInfo.NICType = cns.InfraNIC // assuming we only assign infra nics in this function - ipamResult.interfaceInfo[string(ifInfo.NICType)+fmt.Sprint(i)] = ifInfo + ipamResult.interfaceInfo[string(ifInfo.NICType)+strconv.Itoa(i)] = ifInfo } return ipamResult, nil diff --git a/cni/network/network.go b/cni/network/network.go index 829c6c7d26..d01b5b77ee 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -329,7 +329,7 @@ func addNatIPV6SubnetInfo(nwCfg *cni.NetworkConfig, func (plugin *NetPlugin) addIpamInvoker(ipamAddConfig IPAMAddConfig) (IPAMAddResult, error) { ipamAddResult, err := plugin.ipamInvoker.Add(ipamAddConfig) if err != nil { - return IPAMAddResult{}, err + return IPAMAddResult{}, errors.Wrap(err, "failed to add ipam invoker") } sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam interface: %+v", ipamAddResult.PrettyString())) return ipamAddResult, nil @@ -626,12 +626,11 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // CHECK: why does the cns client deal with apipa?-- "cns client" only used in windows to create apipa err = plugin.nm.EndpointCreate(cnsclient, epInfos) if err != nil { - return err // behavior can change if you don't assign to err prior to returning + return errors.Wrap(err, "failed to create endpoint") // behavior can change if you don't assign to err prior to returning } // telemetry added sendEvent(plugin, fmt.Sprintf("CNI ADD Process succeeded for interfaces: %v", ipamAddResult.PrettyString())) return nil - } func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string { @@ -745,7 +744,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // for secondary (Populate addresses) // initially only for infra nic but now applied to all nic types - var addresses = make([]net.IPNet, len(opt.ifInfo.IPConfigs)) + addresses := make([]net.IPNet, len(opt.ifInfo.IPConfigs)) for i, ipconfig := range opt.ifInfo.IPConfigs { addresses[i] = ipconfig.Address } diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 0423727d77..8fff3150ae 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -382,7 +382,7 @@ func TestIpamAddFail(t *testing.T) { }, wantErr: []bool{false}, wantEndpointErr: true, - wantErrMsg: "Failed to create endpoint: MockEndpointClient Error : Endpoint Error", + wantErrMsg: "failed to create endpoint: MockEndpointClient Error : Endpoint Error", expectedEndpoints: 0, }, } @@ -1186,7 +1186,7 @@ func TestPluginSwiftV2Add(t *testing.T) { IfName: eth0IfName, }, wantErr: true, - wantErrMsg: "IPAM Invoker Add failed with error: delegatedVMNIC fail", + wantErrMsg: "IPAM Invoker Add failed with error: failed to add ipam invoker: delegatedVMNIC fail", }, { name: "SwiftV2 EndpointClient Add fail", @@ -1211,7 +1211,7 @@ func TestPluginSwiftV2Add(t *testing.T) { IfName: eth0IfName, }, wantErr: true, - wantErrMsg: "Failed to create endpoint: MockEndpointClient Error : AddEndpoints Delegated VM NIC failed", + wantErrMsg: "failed to create endpoint: MockEndpointClient Error : AddEndpoints Delegated VM NIC failed", }, } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 601acc53ab..6208397eb0 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -29,7 +29,8 @@ import ( var ( snatConfigFileName = filepath.FromSlash(os.Getenv("TEMP")) + "\\snatConfig" // windows build for version 1903 - win1903Version = 18362 + win1903Version = 18362 + ipv6SubnetInfoCount = 2 ) /* handleConsecutiveAdd handles consecutive add calls for infrastructure containers on Windows platform. @@ -376,7 +377,7 @@ func getLoopbackDSRPolicy(args PolicyArgs) ([]policy.Policy, error) { func getIPV6EndpointPolicy(subnetInfos []network.SubnetInfo) (policy.Policy, error) { var eppolicy policy.Policy - if len(subnetInfos) < 2 { + if len(subnetInfos) < ipv6SubnetInfoCount { return eppolicy, fmt.Errorf("network state doesn't have ipv6 subnet") } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 54ca6bfc4d..45b34adff3 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -418,18 +418,8 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( ep.MacAddress, _ = net.ParseMAC(hnsResponse.MacAddress) - // Confirm with TM: when we delete an endpoint, this code is to find ifName from endpoint and then we can delete this endpoint - // ipconfigs := make([]*IPConfig, len(ep.IPAddresses)) - // for i, ipconfig := range ep.IPAddresses { - // ipconfigs[i] = &IPConfig{Address: ipconfig} - // } - - // // Add secondary interfaces info to CNI state file - // ep.SecondaryInterfaces[ep.IfName] = &InterfaceInfo{ - // MacAddress: ep.MacAddress, - // IPConfigs: ipconfigs, - // Routes: ep.Routes, - // } + // TODO: Confirm with TM: when we delete an endpoint, this code is to find ifName from endpoint and then we can delete this endpoint + // TODO: deal with ep.SecondaryInterfaces here at all anymore? return ep, nil } diff --git a/network/manager_mock.go b/network/manager_mock.go index 224213be75..35e0af9d9e 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -2,7 +2,6 @@ package network import ( "github.com/Azure/azure-container-networking/common" - "github.com/pkg/errors" ) // MockNetworkManager is a mock structure for Network Manager @@ -173,7 +172,7 @@ func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*Endp ep, err := nm.CreateEndpoint(client, epInfo.NetworkID, epInfo) if err != nil { - return errors.Wrap(err, "Failed to create endpoint") + return err } eps = append(eps, ep) } diff --git a/network/network.go b/network/network.go index 9e0af3c681..aa8afdec62 100644 --- a/network/network.go +++ b/network/network.go @@ -10,7 +10,6 @@ import ( "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" - "github.com/pkg/errors" "go.uber.org/zap" ) @@ -327,7 +326,7 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo ep, err := nm.CreateEndpoint(cnsclient, epInfo.NetworkID, epInfo) if err != nil { - return errors.Wrap(err, "Failed to create endpoint") + return err } eps = append(eps, ep) From fec5605e1c30dc5beb2b95baa1fffdc1e693a2de Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 26 Apr 2024 15:59:45 -0700 Subject: [PATCH 062/102] Revert "preserve more logs and reduce timeout for restart for debugging" This reverts commit 0f004925cfb37a4594df63a4259242c37cf07785. --- .pipelines/cni/singletenancy/cniv2-template.yaml | 2 +- cni/log/logger.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/cni/singletenancy/cniv2-template.yaml b/.pipelines/cni/singletenancy/cniv2-template.yaml index 27475670fc..261b44f98c 100644 --- a/.pipelines/cni/singletenancy/cniv2-template.yaml +++ b/.pipelines/cni/singletenancy/cniv2-template.yaml @@ -197,7 +197,7 @@ stages: condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'restartNode') , contains(variables.CONTROL_SCENARIO, 'all') ) ) displayName: "Restart Test" dependsOn: deploy_pods - timeoutInMinutes: 20 # Windows podsubnet takes an extended amount of time to reconcile + timeoutInMinutes: 90 # Windows podsubnet takes an extended amount of time to reconcile steps: - template: ../load-test-templates/restart-node-template.yaml parameters: diff --git a/cni/log/logger.go b/cni/log/logger.go index 8ec6ba39d6..74e4d8bab1 100644 --- a/cni/log/logger.go +++ b/cni/log/logger.go @@ -16,9 +16,9 @@ var ( const ( maxLogFileSizeInMb = 5 - maxLogFileCount = 110 etwCNIEventName = "Azure-CNI" loggingLevel = zapcore.DebugLevel + maxLogFileCount = 8 ) func initZapLog(logFile string) *zap.Logger { From 7fe4df226ea1d5ee86e0664459980b27b7432ecb Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 8 May 2024 11:30:22 -0700 Subject: [PATCH 063/102] ignore error on delete flow network query if we are in stateful cni and do not find the network, we will not error, but when we search for the endpoint it will not be found, leading to us calling ipam invoker delete which is assumed idempotent before returning previously we would error in stateful cni and return before calling ipam invoker delete --- cni/network/network.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index d01b5b77ee..d73deb9ca3 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -1081,9 +1081,6 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { // container runtime tries to delete and create pods which existed before reboot. // this condition will not apply to stateless CNI since the network struct will be crated on each call err = nil - if !plugin.nm.IsStatelessCNIMode() { - return err - } } } // Initialize values from network config. From 03b20423213bc0c408c32d07be800dd0d09f9f12 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 10 May 2024 12:10:09 -0700 Subject: [PATCH 064/102] delete network on endpoint delete if stateless and delegated vmnic (win + linux) --- network/manager.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/network/manager.go b/network/manager.go index b9d1c6ac24..602a06fb40 100644 --- a/network/manager.go +++ b/network/manager.go @@ -475,6 +475,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *EndpointInfo) error { nw := &network{ Id: networkID, // CHECK: currently unused in stateless cni + HnsId: epInfo.HNSNetworkID, Mode: opModeTransparentVlan, SnatBridgeIP: "", extIf: &externalInterface{ @@ -486,6 +487,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint ep := &endpoint{ Id: epInfo.EndpointID, HnsId: epInfo.HNSEndpointID, + HNSNetworkID: epInfo.HNSNetworkID, // unused needed (we use nw.HnsId for deleting the network) HostIfName: epInfo.IfName, LocalIP: "", VlanID: 0, @@ -497,7 +499,18 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint NICType: epInfo.NICType, } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) - return nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) + err := nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) + if err != nil { + return err + } + if epInfo.NICType == cns.DelegatedVMNIC { + // CHECK: should it affect linux? (if it does, it could disconnect external interface, is that okay?) + // bad only when 1) stateless and 2) linux and 3) delegated vmnics exist + logger.Info("Deleting endpoint because delegated vmnic detected", zap.String("HNSNetworkID", nw.HnsId)) + err = nm.deleteNetworkImpl(nw) + // no need to clean up state in stateless + } + return err } // GetEndpointInfo returns information about the given endpoint. From 1c80ac30b5747f4b0e8ed3f2f2a03af4a32d22a7 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 10 May 2024 22:17:36 -0700 Subject: [PATCH 065/102] add nic name, set nicname in linux to master interface name stateless will key into interface map with the nicname field in windows, the nicname field is based on the args ifname (usually eth0) in linux, the nicname field is based on the master interface found (usually eth0) note: hostifname/hostvethname = linux veth pair peer in the host ns ifname/contifname = linux veth pair peer in the container ns, in windows it's just the args ifname nicname is something else ifname isn't used during deletion in linux, hns id is used for deletion in windows --- network/endpoint.go | 6 ++++-- network/endpoint_linux.go | 2 ++ network/endpoint_windows.go | 2 ++ network/manager.go | 13 +++++++------ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index 27de7be7d0..6e9a2f8c65 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -58,6 +58,8 @@ type endpoint struct { SecondaryInterfaces map[string]*InterfaceInfo // Store nic type since we no longer populate SecondaryInterfaces NICType cns.NICType + // Store the key used in stateless + NICName string } // EndpointInfo contains read-only information about an endpoint. @@ -151,9 +153,9 @@ type apipaClient interface { } func (epInfo *EndpointInfo) PrettyString() string { - return fmt.Sprintf("Id:%s ContainerID:%s NetNsPath:%s IfName:%s IfIndex:%d MacAddr:%s IPAddrs:%v Gateways:%v Data:%+v NICType: %s", + return fmt.Sprintf("Id:%s ContainerID:%s NetNsPath:%s IfName:%s IfIndex:%d MacAddr:%s IPAddrs:%v Gateways:%v Data:%+v NICType: %s NetworkContainerID: %s HostIfName: %s", epInfo.EndpointID, epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName, epInfo.IfIndex, epInfo.MacAddress.String(), epInfo.IPAddresses, - epInfo.Gateways, epInfo.Data, epInfo.NICType) + epInfo.Gateways, epInfo.Data, epInfo.NICType, epInfo.NetworkContainerID, epInfo.HostIfName) } func (ifInfo *InterfaceInfo) PrettyString() string { diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index ad65bc69c8..151d233720 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -119,6 +119,8 @@ func (nw *network) newEndpointImpl( Routes: defaultEpInfo.Routes, SecondaryInterfaces: make(map[string]*InterfaceInfo), NICType: defaultEpInfo.NICType, + // Should end up being eth0 in non-delegated nic cases + NICName: defaultEpInfo.MasterIfName, // CHECK: Should find interface name by mac address for linux secondaries } if nw.extIf != nil { ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 88889e3143..fa733f431f 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -171,6 +171,7 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo, plc platform.ExecC NetNs: epInfo.NetNsPath, ContainerID: epInfo.ContainerID, NICType: epInfo.NICType, + NICName: epInfo.IfName, } for _, route := range epInfo.Routes { @@ -410,6 +411,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( PODNameSpace: epInfo.PODNameSpace, // SecondaryInterfaces: make(map[string]*InterfaceInfo), NICType: epInfo.NICType, + NICName: epInfo.IfName, } for _, route := range epInfo.Routes { diff --git a/network/manager.go b/network/manager.go index 602a06fb40..349f9e1589 100644 --- a/network/manager.go +++ b/network/manager.go @@ -487,16 +487,17 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint ep := &endpoint{ Id: epInfo.EndpointID, HnsId: epInfo.HNSEndpointID, - HNSNetworkID: epInfo.HNSNetworkID, // unused needed (we use nw.HnsId for deleting the network) - HostIfName: epInfo.IfName, + HNSNetworkID: epInfo.HNSNetworkID, // unused (we use nw.HnsId for deleting the network) + HostIfName: epInfo.HostIfName, LocalIP: "", VlanID: 0, AllowInboundFromHostToNC: false, AllowInboundFromNCToHost: false, EnableSnatOnHost: false, EnableMultitenancy: false, - NetworkContainerID: epInfo.EndpointID, + NetworkContainerID: epInfo.NetworkContainerID, // we don't use this as long as AllowInboundFromHostToNC and AllowInboundFromNCToHost are false NICType: epInfo.NICType, + NICName: epInfo.IfName, } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) err := nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) @@ -757,7 +758,7 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI NetworkContainerID: endpointID, } - // This is an special case for endpoint state that are being crated by statefull CNI + // If we create an endpoint state with stateful cni and then swap to a stateless cni binary, ifname would not be populated if ifName == "" { ifName = InfraInterfaceName } @@ -765,7 +766,7 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI // filling out the InfraNIC from the state epInfo.IPAddresses = ipInfo.IPv4 epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...) - epInfo.IfName = ifName + epInfo.IfName = ifName // ifname (container veth peer) not used in linux (or even windows) deletion epInfo.HostIfName = ipInfo.HostVethName epInfo.HNSEndpointID = ipInfo.HnsEndpointID epInfo.NICType = ipInfo.NICType @@ -796,7 +797,7 @@ func generateCNSIPInfoMap(eps []*endpoint) map[string]*restserver.IPInfo { ifNametoIPInfoMap := make(map[string]*restserver.IPInfo) // key : interface name, value : IPInfo for _, ep := range eps { - ifNametoIPInfoMap[ep.IfName] = &restserver.IPInfo{ + ifNametoIPInfoMap[ep.NICName] = &restserver.IPInfo{ // in windows, the nicname is args ifname, in linux, it's ethX NICType: ep.NICType, HnsEndpointID: ep.HnsId, HnsNetworkID: ep.HNSNetworkID, From eea03324bb0984caf1607bed4d55abedcc13a8bb Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 14 May 2024 14:01:33 -0700 Subject: [PATCH 066/102] return secondary interface as cni result if no infra nic found, include mac address in cni result --- cni/network/network.go | 9 +++++++-- network/endpoint_windows.go | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index d73deb9ca3..2cec0bcd59 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -404,9 +404,13 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { cniResult := &cniTypesCurr.Result{} for key := range ipamAddResult.interfaceInfo { logger.Info("Exiting add, interface info retrieved", zap.Any("ifInfo", ipamAddResult.interfaceInfo[key])) + // previously we had a default interface info to select which interface info was the one to be returned from cni add + // now we have to infer which interface info should be returned + // we assume that we want to return the infra nic always, and if that is not found, return any one of the secondary interfaces + // if there is an infra nic + secondary, we will always return the infra nic (linux swift v2) + cniResult = convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[key], args.IfName) if ipamAddResult.interfaceInfo[key].NICType == cns.InfraNIC { - cniResult = convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[key], args.IfName) - logger.Info("CNI result generated", zap.Any("cniResult", cniResult)) + break } } @@ -1426,6 +1430,7 @@ func convertInterfaceInfoToCniResult(info network.InterfaceInfo, ifName string) Interfaces: []*cniTypesCurr.Interface{ { Name: ifName, + Mac: info.MacAddress.String(), }, }, DNS: cniTypes.DNS{ diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index fa733f431f..6710987c06 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -338,7 +338,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( } // Create the HCN endpoint. - logger.Info("Creating hcn endpoint", zap.String("name", hcnEndpoint.Name), zap.String("computenetwork", hcnEndpoint.HostComputeNetwork)) + logger.Info("Creating hcn endpoint", zap.Any("hcnEndpoint", hcnEndpoint), zap.String("computenetwork", hcnEndpoint.HostComputeNetwork)) hnsResponse, err := Hnsv2.CreateEndpoint(hcnEndpoint) if err != nil { return nil, fmt.Errorf("Failed to create endpoint: %s due to error: %v", hcnEndpoint.Name, err) From ccf3d8eb8e2d1210d3aec455ebe76a9a9d5adbce Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 15 May 2024 13:58:42 -0700 Subject: [PATCH 067/102] address linter issue --- cni/network/network_test.go | 6 +++--- cnm/network/network.go | 2 +- network/manager.go | 11 ++++++++--- network/manager_mock.go | 12 ++++++------ network/network.go | 2 +- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 8fff3150ae..8f92760e36 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -1042,13 +1042,13 @@ func TestGetAllEndpointState(t *testing.T) { ep2 := getTestEndpoint("podname2", "podnamespace2", "10.0.0.2/24", "podinterfaceid2", "testcontainerid2") ep3 := getTestEndpoint("podname3", "podnamespace3", "10.240.1.242/16", "podinterfaceid3", "testcontainerid3") - _, err := plugin.nm.CreateEndpoint(nil, networkid, ep1) + err := plugin.nm.CreateEndpoint(nil, networkid, ep1) require.NoError(t, err) - _, err = plugin.nm.CreateEndpoint(nil, networkid, ep2) + err = plugin.nm.CreateEndpoint(nil, networkid, ep2) require.NoError(t, err) - _, err = plugin.nm.CreateEndpoint(nil, networkid, ep3) + err = plugin.nm.CreateEndpoint(nil, networkid, ep3) require.NoError(t, err) state, err := plugin.GetAllEndpointState(networkid) diff --git a/cnm/network/network.go b/cnm/network/network.go index 4ae184edfb..94fff34bfe 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -247,7 +247,7 @@ func (plugin *netPlugin) createEndpoint(w http.ResponseWriter, r *http.Request) if err != nil { log.Errorf("failed to init CNS client", err) } - _, err = plugin.nm.CreateEndpoint(cnscli, req.NetworkID, &epInfo) + err = plugin.nm.CreateEndpoint(cnscli, req.NetworkID, &epInfo) // TODO: Because create endpoint no longer assigns to the map or saves to a file, you need to handle it in cnm right here! if err != nil { plugin.SendErrorResponse(w, err) diff --git a/network/manager.go b/network/manager.go index 349f9e1589..7be8b7080d 100644 --- a/network/manager.go +++ b/network/manager.go @@ -101,7 +101,7 @@ type NetworkManager interface { FindNetworkIDFromNetNs(netNs string) (string, error) GetNumEndpointsByContainerID(containerID string) int - CreateEndpoint(client apipaClient, networkID string, epInfo *EndpointInfo) (*endpoint, error) + CreateEndpoint(client apipaClient, networkID string, epInfo *EndpointInfo) error EndpointCreate(client apipaClient, epInfos []*EndpointInfo) error // TODO: change name DeleteEndpoint(networkID string, endpointID string, epInfo *EndpointInfo) error GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) @@ -369,8 +369,7 @@ func (nm *networkManager) GetNetworkInfo(networkID string) (EndpointInfo, error) return nwInfo, nil } -// CreateEndpoint creates a new container endpoint. -func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo *EndpointInfo) (*endpoint, error) { +func (nm *networkManager) createEndpoint(cli apipaClient, networkID string, epInfo *EndpointInfo) (*endpoint, error) { nm.Lock() defer nm.Unlock() @@ -405,6 +404,12 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn return ep, nil } +// CreateEndpoint creates a new container endpoint (this is for compatibility-- add flow should no longer use this). +func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo *EndpointInfo) error { + _, err := nm.createEndpoint(cli, networkID, epInfo) + return err +} + // UpdateEndpointState will make a call to CNS updatEndpointState API in the stateless CNI mode // It will add HNSEndpointID or HostVeth name to the endpoint state func (nm *networkManager) UpdateEndpointState(eps []*endpoint) error { diff --git a/network/manager_mock.go b/network/manager_mock.go index 35e0af9d9e..c847a1a2c2 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -54,13 +54,13 @@ func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (EndpointInfo, er // CreateEndpoint mock // TODO: Fix mock behavior because create endpoint no longer also saves the state -func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfo *EndpointInfo) (*endpoint, error) { +func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfo *EndpointInfo) error { if err := nm.TestEndpointClient.AddEndpoints(epInfo); err != nil { - return nil, err + return err } nm.TestEndpointInfoMap[epInfo.EndpointID] = epInfo - return &endpoint{}, nil + return nil } // DeleteEndpoint mock @@ -170,14 +170,14 @@ func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*Endp } } - ep, err := nm.CreateEndpoint(client, epInfo.NetworkID, epInfo) + err := nm.CreateEndpoint(client, epInfo.NetworkID, epInfo) if err != nil { return err } - eps = append(eps, ep) + eps = append(eps, &endpoint{}) // mock append } - // save endpoints + // mock save endpoints return nm.SaveState(eps) } diff --git a/network/network.go b/network/network.go index aa8afdec62..b82d56ad27 100644 --- a/network/network.go +++ b/network/network.go @@ -324,7 +324,7 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo } } - ep, err := nm.CreateEndpoint(cnsclient, epInfo.NetworkID, epInfo) + ep, err := nm.createEndpoint(cnsclient, epInfo.NetworkID, epInfo) if err != nil { return err } From 8a2ae3feed7be5a0134c4aca98dcffc880c7f912 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 17 May 2024 12:15:46 -0700 Subject: [PATCH 068/102] fix critical error where failing to add in windows stateless would lead to hns components not being deleted and add netns for hnsv2 tested by triggering a failure to save the stateless state and seeing that the hns endpoint and network are cleaned up we use the endpoint info to clean up on "add" error, but previously, we didn't populate it with the hns ids to do so adds netns to stateless as the presence of a valid guid in netns determines if hnsv2 is used --- network/endpoint.go | 4 ++-- network/endpoint_windows.go | 4 ++++ network/manager.go | 4 +++- network/network_windows.go | 4 ++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index 6e9a2f8c65..0087bdf8eb 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -153,9 +153,9 @@ type apipaClient interface { } func (epInfo *EndpointInfo) PrettyString() string { - return fmt.Sprintf("Id:%s ContainerID:%s NetNsPath:%s IfName:%s IfIndex:%d MacAddr:%s IPAddrs:%v Gateways:%v Data:%+v NICType: %s NetworkContainerID: %s HostIfName: %s", + return fmt.Sprintf("Id:%s ContainerID:%s NetNsPath:%s IfName:%s IfIndex:%d MacAddr:%s IPAddrs:%v Gateways:%v Data:%+v NICType: %s NetworkContainerID: %s HostIfName: %s NetNs: %s", epInfo.EndpointID, epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName, epInfo.IfIndex, epInfo.MacAddress.String(), epInfo.IPAddresses, - epInfo.Gateways, epInfo.Data, epInfo.NICType, epInfo.NetworkContainerID, epInfo.HostIfName) + epInfo.Gateways, epInfo.Data, epInfo.NICType, epInfo.NetworkContainerID, epInfo.HostIfName, epInfo.NetNs) } func (ifInfo *InterfaceInfo) PrettyString() string { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 6710987c06..72fd1f4223 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -180,6 +180,8 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo, plc platform.ExecC ep.MacAddress, _ = net.ParseMAC(hnsResponse.MacAddress) + epInfo.HNSEndpointID = hnsResponse.Id // we use the ep info hns id later in stateless to clean up in ADD if there is an error + return ep, nil } @@ -420,6 +422,8 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( ep.MacAddress, _ = net.ParseMAC(hnsResponse.MacAddress) + epInfo.HNSEndpointID = hnsResponse.Id // we use the ep info hns id later in stateless to clean up in ADD if there is an error + // TODO: Confirm with TM: when we delete an endpoint, this code is to find ifName from endpoint and then we can delete this endpoint // TODO: deal with ep.SecondaryInterfaces here at all anymore? diff --git a/network/manager.go b/network/manager.go index 7be8b7080d..200bc22197 100644 --- a/network/manager.go +++ b/network/manager.go @@ -483,6 +483,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint HnsId: epInfo.HNSNetworkID, Mode: opModeTransparentVlan, SnatBridgeIP: "", + NetNs: epInfo.NetNs, // to trigger hns v2, windows extIf: &externalInterface{ Name: InfraInterfaceName, MacAddress: nil, @@ -496,11 +497,12 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint HostIfName: epInfo.HostIfName, LocalIP: "", VlanID: 0, - AllowInboundFromHostToNC: false, + AllowInboundFromHostToNC: false, // stateless currently does not support apipa AllowInboundFromNCToHost: false, EnableSnatOnHost: false, EnableMultitenancy: false, NetworkContainerID: epInfo.NetworkContainerID, // we don't use this as long as AllowInboundFromHostToNC and AllowInboundFromNCToHost are false + NetNs: epInfo.NetNs, // to trigger hnsv2, windows NICType: epInfo.NICType, NICName: epInfo.IfName, } diff --git a/network/network_windows.go b/network/network_windows.go index fbfcc5d142..6d7feaf49d 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -183,6 +183,8 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *EndpointInfo, extIf *exter NetNs: nwInfo.NetNs, } + nwInfo.HNSNetworkID = hnsResponse.Id // we use this later in stateless to clean up in ADD if there is an error + globals, err := Hnsv1.GetHNSGlobals() if err != nil || globals.Version.Major <= hcsshim.HNSVersion1803.Major { // err would be not nil for windows 1709 & below @@ -416,6 +418,8 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *EndpointInfo, extIf *exter NetNs: nwInfo.NetNs, } + nwInfo.HNSNetworkID = hnsResponse.Id // we use this later in stateless to clean up in ADD if there is an error + return nw, nil } From b6eb3426465320a2ca3942ef2724944cace8d078 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 17 May 2024 15:59:08 -0700 Subject: [PATCH 069/102] set nicname used in stateless cni according to feedback --- network/endpoint_linux.go | 8 +++++++- network/endpoint_windows.go | 8 +++++++- network/manager.go | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 151d233720..73da43ca79 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -98,6 +98,12 @@ func (nw *network) newEndpointImpl( contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, defaultEpInfo.EndpointID[:7]) } + nicName := epInfo.IfName + // infra nic nicname will look like eth0, but delegated/secondary nics will look like "eth1 eth2 eth3" + if epInfo.NICType != cns.InfraNIC { + nicName = epInfo.MasterIfName + } + ep := &endpoint{ Id: defaultEpInfo.EndpointID, IfName: contIfName, // container veth pair name. In cnm, we won't rename this and docker expects veth name. @@ -120,7 +126,7 @@ func (nw *network) newEndpointImpl( SecondaryInterfaces: make(map[string]*InterfaceInfo), NICType: defaultEpInfo.NICType, // Should end up being eth0 in non-delegated nic cases - NICName: defaultEpInfo.MasterIfName, // CHECK: Should find interface name by mac address for linux secondaries + NICName: nicName, // CHECK: Should find interface name by mac address for linux secondaries } if nw.extIf != nil { ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 72fd1f4223..66f291dfe2 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -393,6 +393,12 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) } + nicName := epInfo.IfName + // infra nic nicname will look like eth0, but delegated/secondary nics will look like "vEthernet 2" + if epInfo.NICType != cns.InfraNIC { + nicName = epInfo.MasterIfName + } + // Create the endpoint object. ep := &endpoint{ Id: hcnEndpoint.Name, @@ -413,7 +419,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( PODNameSpace: epInfo.PODNameSpace, // SecondaryInterfaces: make(map[string]*InterfaceInfo), NICType: epInfo.NICType, - NICName: epInfo.IfName, + NICName: nicName, } for _, route := range epInfo.Routes { diff --git a/network/manager.go b/network/manager.go index 200bc22197..bd5acca411 100644 --- a/network/manager.go +++ b/network/manager.go @@ -700,7 +700,7 @@ func (nm *networkManager) GetNumberOfEndpoints(ifName string, networkId string) // GetEndpointID returns a unique endpoint ID based on the CNI mode. func (nm *networkManager) GetEndpointID(containerID, ifName string) string { if nm.IsStatelessCNIMode() { - return containerID // CHECK: In stateless cni do we need to also add the num or is it always the same ep id for all endpoints with the same container id? + return containerID } if len(containerID) > ContainerIDLength { containerID = containerID[:ContainerIDLength] From 3d23c6d487cf1fffc33f4a3da63705859dd6d545 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 17 May 2024 16:58:56 -0700 Subject: [PATCH 070/102] add dummy guid to stateless delete since we assume stateless is always hnsv2 we assume that the netns value isn't used in stateless deletion --- network/manager.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/network/manager.go b/network/manager.go index bd5acca411..37835ffdfd 100644 --- a/network/manager.go +++ b/network/manager.go @@ -39,6 +39,7 @@ const ( ContainerIDLength = 8 EndpointIfIndex = 0 // Azure CNI supports only one interface DefaultNetworkID = "azure" + dummyGUID = "12345678-1234-1234-1234-123456789012" ) var Ipv4DefaultRouteDstPrefix = net.IPNet{ @@ -483,7 +484,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint HnsId: epInfo.HNSNetworkID, Mode: opModeTransparentVlan, SnatBridgeIP: "", - NetNs: epInfo.NetNs, // to trigger hns v2, windows + NetNs: dummyGUID, // to trigger hns v2, windows extIf: &externalInterface{ Name: InfraInterfaceName, MacAddress: nil, @@ -502,7 +503,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint EnableSnatOnHost: false, EnableMultitenancy: false, NetworkContainerID: epInfo.NetworkContainerID, // we don't use this as long as AllowInboundFromHostToNC and AllowInboundFromNCToHost are false - NetNs: epInfo.NetNs, // to trigger hnsv2, windows + NetNs: dummyGUID, // to trigger hnsv2, windows NICType: epInfo.NICType, NICName: epInfo.IfName, } From 5356b4da143009ba2cd5b8b527195fe8480c2bbd Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 17 May 2024 18:00:52 -0700 Subject: [PATCH 071/102] clean up createEpInfo, declare endpoint info once --- cni/network/network.go | 114 +++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 68 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 2cec0bcd59..0b6a5d6b7c 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -674,8 +674,7 @@ type createEpInfoOpt struct { func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointInfo, error) { // you can modify to pass in whatever else you need var ( - epInfo network.EndpointInfo - nwInfo network.EndpointInfo + endpointInfo network.EndpointInfo ) // ensure we can find the master interface opt.ifInfo.HostSubnetPrefix.IP = opt.ifInfo.HostSubnetPrefix.IP.Mask(opt.ifInfo.HostSubnetPrefix.Mask) @@ -692,30 +691,26 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn return nil, err } // this struct is for organization and represents fields from the network to be merged into the endpoint info later - nwInfo = network.EndpointInfo{ + endpointInfo = network.EndpointInfo{ NetworkID: opt.networkID, Mode: opt.ipamAddConfig.nwCfg.Mode, MasterIfName: masterIfName, AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, BridgeName: opt.ipamAddConfig.nwCfg.Bridge, - EnableSnatOnHost: opt.ipamAddConfig.nwCfg.EnableSnatOnHost, // (unused) same value as field with same name in epInfo above - NetworkDNS: nwDNSInfo, // (unused) nw and ep dns infos are separated to avoid possible conflicts - Policies: opt.policies, // present infra only + NetworkDNS: nwDNSInfo, // nw and ep dns infos are separated to avoid possible conflicts + Policies: opt.policies, // CHECK: Should there be endpoint policies and a separate network policies? NetNs: opt.ipamAddConfig.args.Netns, Options: opt.ipamAddConfig.options, DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - IPV6Mode: opt.ipamAddConfig.nwCfg.IPV6Mode, // present infra only TODO: check if IPV6Mode field can be deprecated (unused) - IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, // present infra only - ServiceCidrs: opt.ipamAddConfig.nwCfg.ServiceCidrs, // (unused) same value as field with same name in epInfo above - IsIPv6Enabled: opt.ipv6Enabled, // present infra only - NICType: opt.ifInfo.NICType, + IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, // present infra only + IsIPv6Enabled: opt.ipv6Enabled, // present infra only } - if err = addSubnetToNetworkInfo(*opt.ifInfo, &nwInfo); err != nil { - logger.Info("Failed to add subnets to networkInfo", zap.Error(err)) + if err = addSubnetToEndpointInfo(*opt.ifInfo, &endpointInfo); err != nil { + logger.Info("Failed to add subnets to endpointInfo", zap.Error(err)) return nil, err } - setNetworkOptions(opt.ifInfo.NCResponse, &nwInfo) + setNetworkOptions(opt.ifInfo.NCResponse, &endpointInfo) // populate endpoint info epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, opt.ifInfo.DNS, opt.k8sNamespace) // Probably won't panic if given bad values @@ -725,7 +720,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn return nil, err } policyArgs := PolicyArgs{ - subnetInfos: nwInfo.Subnets, // getEndpointPolicies requires nwInfo.Subnets only (checked) + subnetInfos: endpointInfo.Subnets, // getEndpointPolicies requires nwInfo.Subnets only (checked) nwCfg: opt.nwCfg, ipconfigs: opt.ifInfo.IPConfigs, } @@ -743,7 +738,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. // IT will result in unpredictable behavior if API server decides to // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", nwInfo.NetworkID, opt.args.ContainerID, opt.args.IfName) + vethName = fmt.Sprintf("%s%s%s", endpointInfo.NetworkID, opt.args.ContainerID, opt.args.IfName) } // for secondary (Populate addresses) @@ -763,77 +758,60 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn endpointID = plugin.nm.GetEndpointID(opt.args.ContainerID, strconv.Itoa(opt.idx)) } - epInfo = network.EndpointInfo{ - EndpointID: endpointID, - ContainerID: opt.args.ContainerID, - NetNsPath: opt.args.Netns, - IfName: opt.args.IfName, - Data: make(map[string]interface{}), - EndpointDNS: epDNSInfo, - Policies: opt.policies, - IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, - EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, - EnableMultiTenancy: opt.nwCfg.MultiTenancy, - EnableInfraVnet: opt.enableInfraVnet, - EnableSnatForDns: opt.enableSnatForDNS, - PODName: opt.k8sPodName, - PODNameSpace: opt.k8sNamespace, - SkipHotAttachEp: false, // Hot attach at the time of endpoint creation - IPV6Mode: opt.nwCfg.IPV6Mode, - VnetCidrs: opt.nwCfg.VnetCidrs, - ServiceCidrs: opt.nwCfg.ServiceCidrs, - NATInfo: opt.natInfo, - NICType: opt.ifInfo.NICType, - SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, - Routes: opt.ifInfo.Routes, - // added the following for delegated vm nic - IPAddresses: addresses, - MacAddress: opt.ifInfo.MacAddress, - // the following is used for creating an external interface if we can't find an existing network - HostSubnetPrefix: opt.ifInfo.HostSubnetPrefix.String(), - } + endpointInfo.EndpointID = endpointID + endpointInfo.ContainerID = opt.args.ContainerID + endpointInfo.NetNsPath = opt.args.Netns // probably same value as epInfo.NetNs + endpointInfo.IfName = opt.args.IfName + endpointInfo.Data = make(map[string]interface{}) + endpointInfo.EndpointDNS = epDNSInfo + endpointInfo.Policies = opt.policies + endpointInfo.IPsToRouteViaHost = opt.nwCfg.IPsToRouteViaHost + endpointInfo.EnableSnatOnHost = opt.nwCfg.EnableSnatOnHost + endpointInfo.EnableMultiTenancy = opt.nwCfg.MultiTenancy + endpointInfo.EnableInfraVnet = opt.enableInfraVnet + endpointInfo.EnableSnatForDns = opt.enableSnatForDNS + endpointInfo.PODName = opt.k8sPodName + endpointInfo.PODNameSpace = opt.k8sNamespace + endpointInfo.SkipHotAttachEp = false // Hot attach at the time of endpoint creation + endpointInfo.IPV6Mode = opt.nwCfg.IPV6Mode + endpointInfo.VnetCidrs = opt.nwCfg.VnetCidrs + endpointInfo.ServiceCidrs = opt.nwCfg.ServiceCidrs + endpointInfo.NATInfo = opt.natInfo + endpointInfo.NICType = opt.ifInfo.NICType + endpointInfo.SkipDefaultRoutes = opt.ifInfo.SkipDefaultRoutes + endpointInfo.Routes = opt.ifInfo.Routes + // added the following for delegated vm nic + endpointInfo.IPAddresses = addresses + endpointInfo.MacAddress = opt.ifInfo.MacAddress + // the following is used for creating an external interface if we can't find an existing network + endpointInfo.HostSubnetPrefix = opt.ifInfo.HostSubnetPrefix.String() epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) // not specific to delegated or infra if err != nil { logger.Error("failed to get policies from runtime configurations", zap.Error(err)) return nil, plugin.Errorf(err.Error()) } - epInfo.Policies = append(epInfo.Policies, epPolicies...) + endpointInfo.Policies = append(endpointInfo.Policies, epPolicies...) if opt.ipamAddResult.ipv6Enabled { // not specific to this particular interface - epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working + endpointInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working } if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { - epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address + endpointInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address } if opt.nwCfg.MultiTenancy { // previously only infra nic was passed into this function but now all nics are passed in (possibly breaks swift v2) - plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, opt.ifInfo) + plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &endpointInfo, opt.ifInfo) } - setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) - - // populate endpoint info with network info fields - epInfo.MasterIfName = nwInfo.MasterIfName - epInfo.AdapterName = nwInfo.AdapterName - epInfo.NetworkID = nwInfo.NetworkID - epInfo.Mode = nwInfo.Mode - epInfo.Subnets = nwInfo.Subnets - epInfo.PodSubnet = nwInfo.PodSubnet - epInfo.BridgeName = nwInfo.BridgeName - epInfo.NetNs = nwInfo.NetNs - epInfo.Options = nwInfo.Options - epInfo.DisableHairpinOnHostInterface = nwInfo.DisableHairpinOnHostInterface - epInfo.IPAMType = nwInfo.IPAMType - epInfo.IsIPv6Enabled = nwInfo.IsIPv6Enabled - epInfo.NetworkDNS = nwInfo.NetworkDNS + setEndpointOptions(opt.cnsNetworkConfig, &endpointInfo, vethName) - logger.Info("Generated endpoint info from fields", zap.String("epInfo", epInfo.PrettyString())) + logger.Info("Generated endpoint info from fields", zap.String("epInfo", endpointInfo.PrettyString())) // now our ep info should have the full combined information from both the network and endpoint structs - return &epInfo, nil + return &endpointInfo, nil } // cleanup allocated ipv4 and ipv6 addresses if they exist @@ -863,7 +841,7 @@ func (plugin *NetPlugin) getNetworkDNSSettings(nwCfg *cni.NetworkConfig, dns net } // construct network info with ipv4/ipv6 subnets -func addSubnetToNetworkInfo(interfaceInfo network.InterfaceInfo, nwInfo *network.EndpointInfo) error { +func addSubnetToEndpointInfo(interfaceInfo network.InterfaceInfo, nwInfo *network.EndpointInfo) error { for _, ipConfig := range interfaceInfo.IPConfigs { ip, podSubnetPrefix, err := net.ParseCIDR(ipConfig.Address.String()) if err != nil { From 6df1b9f44f9dbb6367db188219d9a39a4c9b4f54 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 20 May 2024 15:11:29 -0700 Subject: [PATCH 072/102] address feedback from vipul --- cni/log/logger.go | 2 +- cni/network/invoker_azure.go | 10 +++++++--- cni/network/invoker_cns.go | 32 ++++++++++++++++++++------------ cni/network/multitenancy.go | 6 +++++- cni/network/network.go | 23 ++++++++++++++++++----- 5 files changed, 51 insertions(+), 22 deletions(-) diff --git a/cni/log/logger.go b/cni/log/logger.go index 74e4d8bab1..1f47a4ec99 100644 --- a/cni/log/logger.go +++ b/cni/log/logger.go @@ -16,9 +16,9 @@ var ( const ( maxLogFileSizeInMb = 5 + maxLogFileCount = 8 etwCNIEventName = "Azure-CNI" loggingLevel = zapcore.DebugLevel - maxLogFileCount = 8 ) func initZapLog(logFile string) *zap.Logger { diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index 91057ff887..6cd3940eff 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -72,8 +72,8 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er defer func() { if err != nil { - if len(addResult.interfaceInfo) > 0 && len(addResult.interfaceInfo[string(cns.InfraNIC)].IPConfigs) > 0 { - if er := invoker.Delete(&addResult.interfaceInfo[string(cns.InfraNIC)].IPConfigs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { + if len(addResult.interfaceInfo) > 0 && len(addResult.interfaceInfo[invoker.getInterfaceInfoKey(cns.InfraNIC)].IPConfigs) > 0 { + if er := invoker.Delete(&addResult.interfaceInfo[invoker.getInterfaceInfoKey(cns.InfraNIC)].IPConfigs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { err = invoker.plugin.Errorf("Failed to clean up IP's during Delete with error %v, after Add failed with error %w", er, err) } } else { @@ -118,7 +118,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er if len(result.IPs) > 0 { hostSubnetPrefix = result.IPs[0].Address } - addResult.interfaceInfo[string(cns.InfraNIC)] = network.InterfaceInfo{ + addResult.interfaceInfo[invoker.getInterfaceInfoKey(cns.InfraNIC)] = network.InterfaceInfo{ IPConfigs: ipconfigs, Routes: routes, DNS: network.DNSInfo{ @@ -208,3 +208,7 @@ func (invoker *AzureIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkCo return nil } + +func (invoker *AzureIPAMInvoker) getInterfaceInfoKey(nicType cns.NICType) string { + return string(nicType) +} diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 1f6bf4d985..60c433aee7 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -164,6 +164,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro //nolint:exhaustive // ignore exhaustive types check // Do we want to leverage this lint skip in other places of our code? + key := invoker.getInterfaceInfoKey(info.nicType, info.macAddress) switch info.nicType { case cns.DelegatedVMNIC: // only handling single v4 PodIPInfo for DelegatedVMNICs at the moment, will have to update once v6 gets added @@ -176,25 +177,25 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro info.hostPrimaryIP = response.PodIPInfo[i].HostPrimaryIPInfo.PrimaryIP info.hostGateway = response.PodIPInfo[i].HostPrimaryIPInfo.Gateway - if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig); err != nil { + if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig, key); err != nil { return IPAMAddResult{}, err } case cns.InfraNIC: // only count dualstack interface once - _, exist := addResult.interfaceInfo[string(info.nicType)] + _, exist := addResult.interfaceInfo[key] if !exist { - addResult.interfaceInfo[string(info.nicType)] = network.InterfaceInfo{} + addResult.interfaceInfo[key] = network.InterfaceInfo{} if !info.skipDefaultRoutes { numInterfacesWithDefaultRoutes++ } } overlayMode := (invoker.ipamMode == util.V4Overlay) || (invoker.ipamMode == util.DualStackOverlay) || (invoker.ipamMode == util.Overlay) - if err := configureDefaultAddResult(&info, &addConfig, &addResult, overlayMode); err != nil { + if err := configureDefaultAddResult(&info, &addConfig, &addResult, overlayMode, key); err != nil { return IPAMAddResult{}, err } default: - // TODO Error check or continue? + logger.Info("Unknown NIC type received from cns pod ip info", zap.String("nicType", string(info.nicType))) } } @@ -361,7 +362,7 @@ func getRoutes(cnsRoutes []cns.Route, skipDefaultRoutes bool) ([]network.RouteIn return routes, nil } -func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, addResult *IPAMAddResult, overlayMode bool) error { +func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, addResult *IPAMAddResult, overlayMode bool, key string) error { // set the NC Primary IP in options // SNATIPKey is not set for ipv6 if net.ParseIP(info.ncPrimaryIP).To4() != nil { @@ -401,7 +402,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add // get the name of the primary IP address _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) if err != nil { - return fmt.Errorf("unable to parse hostSubnet: %w", err) + return errors.Wrap(err, "unable to parse hostSubnet") } if ip := net.ParseIP(info.podIPAddress); ip != nil { @@ -411,7 +412,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add addResult.ipv6Enabled = true } - ipConfigs := addResult.interfaceInfo[string(info.nicType)].IPConfigs + ipConfigs := addResult.interfaceInfo[key].IPConfigs ipConfigs = append(ipConfigs, &network.IPConfig{ Address: net.IPNet{ @@ -426,7 +427,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add return getRoutesErr } - resRoute := addResult.interfaceInfo[string(info.nicType)].Routes + resRoute := addResult.interfaceInfo[key].Routes if len(routes) > 0 { resRoute = append(resRoute, routes...) } else { // add default routes if none are provided @@ -437,7 +438,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } // if we have multiple infra ip result infos, we effectively append routes and ip configs to that same interface info each time // the host subnet prefix (in ipv4 or ipv6) will always refer to the same interface regardless of which ip result info we look at - addResult.interfaceInfo[string(info.nicType)] = network.InterfaceInfo{ + addResult.interfaceInfo[key] = network.InterfaceInfo{ NICType: cns.InfraNIC, SkipDefaultRoutes: info.skipDefaultRoutes, IPConfigs: ipConfigs, @@ -458,7 +459,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add return nil } -func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, podIPConfig *cns.IPSubnet) error { +func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, podIPConfig *cns.IPSubnet, key string) error { ip, ipnet, err := podIPConfig.GetIPNet() if ip == nil { return errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") @@ -474,7 +475,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p return err } - addResult.interfaceInfo[string(info.macAddress)] = network.InterfaceInfo{ + addResult.interfaceInfo[key] = network.InterfaceInfo{ IPConfigs: []*network.IPConfig{ { Address: net.IPNet{ @@ -492,3 +493,10 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p return nil } + +func (invoker *CNSIPAMInvoker) getInterfaceInfoKey(nicType cns.NICType, macAddress string) string { + if nicType == cns.DelegatedVMNIC { + return macAddress + } + return string(nicType) +} diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 6481d241f7..190865795b 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -232,7 +232,7 @@ func (m *Multitenancy) GetAllNetworkContainers( ifInfo.NICType = cns.InfraNIC // assuming we only assign infra nics in this function - ipamResult.interfaceInfo[string(ifInfo.NICType)+strconv.Itoa(i)] = ifInfo + ipamResult.interfaceInfo[m.getInterfaceInfoKey(ifInfo.NICType, i)] = ifInfo } return ipamResult, err @@ -366,6 +366,10 @@ func checkIfSubnetOverlaps(enableInfraVnet bool, nwCfg *cni.NetworkConfig, cnsNe return false } +func (m *Multitenancy) getInterfaceInfoKey(nicType cns.NICType, i int) string { + return string(nicType) + strconv.Itoa(i) +} + var ( errSnatIP = errors.New("Snat IP not populated") errInfraVnet = errors.New("infravnet not populated") diff --git a/cni/network/network.go b/cni/network/network.go index 0b6a5d6b7c..b6716f351f 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -207,14 +207,21 @@ func (plugin *NetPlugin) Stop() { // findInterfaceByMAC returns the name of the master interface func (plugin *NetPlugin) findInterfaceByMAC(macAddress string) string { - interfaces, _ := net.Interfaces() + interfaces, err := net.Interfaces() + if err != nil { + logger.Error("failed to get interfaces", zap.Error(err)) + return "" + } + macs := make([]string, 0, len(interfaces)) for _, iface := range interfaces { // find master interface by macAddress for Swiftv2 + macs = append(macs, iface.HardwareAddr.String()) if iface.HardwareAddr.String() == macAddress { return iface.Name } } // Failed to find a suitable interface. + logger.Error("Failed to find interface by MAC", zap.String("macAddress", macAddress), zap.Strings("interfaces", macs)) return "" } @@ -228,7 +235,12 @@ func (plugin *NetPlugin) findMasterInterfaceBySubnet(nwCfg *cni.NetworkConfig, s // Otherwise, pick the first interface with an IP address in the given subnet. subnetPrefixString := subnetPrefix.String() - interfaces, _ := net.Interfaces() + interfaces, err := net.Interfaces() + if err != nil { + logger.Error("failed to get interfaces", zap.Error(err)) + return "" + } + var ipnets []string for _, iface := range interfaces { addrs, _ := iface.Addrs() for _, addr := range addrs { @@ -236,6 +248,7 @@ func (plugin *NetPlugin) findMasterInterfaceBySubnet(nwCfg *cni.NetworkConfig, s if err != nil { continue } + ipnets = append(ipnets, ipnet.String()) if subnetPrefixString == ipnet.String() { return iface.Name } @@ -243,6 +256,7 @@ func (plugin *NetPlugin) findMasterInterfaceBySubnet(nwCfg *cni.NetworkConfig, s } // Failed to find a suitable interface. + logger.Error("Failed to find interface by subnet prefix", zap.String("subnetPrefix", subnetPrefixString), zap.Strings("interfaces", ipnets)) return "" } @@ -492,6 +506,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options} if nwCfg.MultiTenancy { + // triggered only in swift v1 multitenancy // dual nic multitenancy -> two interface infos // multitenancy (swift v1) -> one interface info plugin.report.Context = "AzureCNIMultitenancy" @@ -673,9 +688,7 @@ type createEpInfoOpt struct { } func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointInfo, error) { // you can modify to pass in whatever else you need - var ( - endpointInfo network.EndpointInfo - ) + var endpointInfo network.EndpointInfo // ensure we can find the master interface opt.ifInfo.HostSubnetPrefix.IP = opt.ifInfo.HostSubnetPrefix.IP.Mask(opt.ifInfo.HostSubnetPrefix.Mask) opt.ipamAddConfig.nwCfg.IPAM.Subnet = opt.ifInfo.HostSubnetPrefix.String() From 182a3df5ed7dc65c7787e98c2695a44ef1e45bd9 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 21 May 2024 09:13:47 -0700 Subject: [PATCH 073/102] change comments only --- cni/network/network.go | 12 +++++------- network/manager.go | 3 ++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index b6716f351f..ac4100d46c 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -1062,16 +1062,14 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { // deleted, getNetworkName will return error of the type NetworkNotFoundError which will result in nil error as compliance // with CNI SPEC as mentioned below. - // CHECK: args.Netns or epInfo.Netns (it's not populated when we convert ep to epInfo) - // CHECK: When do we have multiple network ids? + // We get the network id and nw info here to preserve existing behavior networkID, err = plugin.getNetworkID(args.Netns, nil, nwCfg) - // CHECK: When do we get multiple network infos if nwInfo, err = plugin.nm.GetNetworkInfo(networkID); err != nil { if !nwCfg.MultiTenancy { logger.Error("Failed to query network", zap.String("network", networkID), zap.Error(err)) - // Log the error but return success if the network is not found. + // Log the error if the network is not found. // if cni hits this, mostly state file would be missing and it can be reboot scenario where // container runtime tries to delete and create pods which existed before reboot. // this condition will not apply to stateless CNI since the network struct will be crated on each call @@ -1094,8 +1092,9 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { var epInfos []*network.EndpointInfo if plugin.nm.IsStatelessCNIMode() { + // network ID is passed in and used only for migration + // otherwise, in stateless, we don't need the network id for deletion epInfos, err = plugin.nm.GetEndpointState(networkID, args.ContainerID) - // TODO: epInfos from stateless cni does not have networkID populated and so DeleteEndpoint will not have the networkID } else { epInfos = plugin.nm.GetEndpointInfosFromContainerID(args.ContainerID) } @@ -1125,8 +1124,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { // populate ep infos here in loop if necessary // delete endpoints for _, epInfo := range epInfos { - // CHECK: in stateless, network id is not populated in epInfo, but in stateful cni, it is (nw id is used in stateful) - // CHECK: in stateless cni, we do not use the network id, but pass in network id to cover stateful case + // in stateless, network id is not populated in epInfo, but in stateful cni, it is (nw id is used in stateful) if err = plugin.nm.DeleteEndpoint(epInfo.NetworkID, epInfo.EndpointID, epInfo); err != nil { // An error will not be returned if the endpoint is not found // return a retriable error so the container runtime will retry this DEL later diff --git a/network/manager.go b/network/manager.go index 37835ffdfd..5a6269ec6b 100644 --- a/network/manager.go +++ b/network/manager.go @@ -513,7 +513,8 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint return err } if epInfo.NICType == cns.DelegatedVMNIC { - // CHECK: should it affect linux? (if it does, it could disconnect external interface, is that okay?) + // we are currently assuming stateless is not running in linux + // CHECK: could this affect linux? (if it does, it could disconnect external interface, is that okay?) // bad only when 1) stateless and 2) linux and 3) delegated vmnics exist logger.Info("Deleting endpoint because delegated vmnic detected", zap.String("HNSNetworkID", nw.HnsId)) err = nm.deleteNetworkImpl(nw) From b9ea97d22cde0ad62669170e4902b7efa9efdd11 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 21 May 2024 11:40:18 -0700 Subject: [PATCH 074/102] revert change to cns package --- cns/client/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cns/client/client.go b/cns/client/client.go index 6815349e8e..804580b8d8 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1023,11 +1023,11 @@ func (c *Client) GetHomeAz(ctx context.Context) (*cns.GetHomeAzResponse, error) return &getHomeAzResponse, nil } -// GetEndpoint calls the EndpointHandlerAPI in CNS to retrieve the state of a given containerID -func (c *Client) GetEndpoint(ctx context.Context, containerID string) (*restserver.GetEndpointResponse, error) { +// GetEndpoint calls the EndpointHandlerAPI in CNS to retrieve the state of a given EndpointID +func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*restserver.GetEndpointResponse, error) { // build the request u := c.routes[cns.EndpointAPI] - uString := u.String() + containerID + uString := u.String() + endpointID req, err := http.NewRequestWithContext(ctx, http.MethodGet, uString, http.NoBody) if err != nil { return nil, errors.Wrap(err, "failed to build request") From 270c11303f43337dbdd1818be3a4e3cc1fdfc5b9 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 21 May 2024 15:18:32 -0700 Subject: [PATCH 075/102] fix stateless cni migration flow not having nictype on migrate --- network/manager.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/network/manager.go b/network/manager.go index 5a6269ec6b..e7f418128e 100644 --- a/network/manager.go +++ b/network/manager.go @@ -768,14 +768,17 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI } // If we create an endpoint state with stateful cni and then swap to a stateless cni binary, ifname would not be populated + // triggered in migration to stateless only, assuming no incomplete state for delegated if ifName == "" { ifName = InfraInterfaceName + ipInfo.NICType = cns.InfraNIC } // filling out the InfraNIC from the state epInfo.IPAddresses = ipInfo.IPv4 epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...) - epInfo.IfName = ifName // ifname (container veth peer) not used in linux (or even windows) deletion + epInfo.IfName = ifName // epInfo.IfName is set to the value of the NICName when the endpoint was added + // sidenote: ifname doesn't seem to be used in linux (or even windows) deletion epInfo.HostIfName = ipInfo.HostVethName epInfo.HNSEndpointID = ipInfo.HnsEndpointID epInfo.NICType = ipInfo.NICType From f5e939bacd7228dd07a9c36c3bb19d7b890a51df Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 21 May 2024 15:23:34 -0700 Subject: [PATCH 076/102] keep nwInfo variables named the same as before pr (noop) --- network/bridge_networkclient_linux.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go index c66cffce3d..c5159b2643 100644 --- a/network/bridge_networkclient_linux.go +++ b/network/bridge_networkclient_linux.go @@ -25,7 +25,7 @@ func newErrorLinuxBridgeClient(errStr string) error { type LinuxBridgeClient struct { bridgeName string hostInterfaceName string - epInfo EndpointInfo + nwInfo EndpointInfo netlink netlink.NetlinkInterface nuClient networkutils.NetworkUtils } @@ -33,13 +33,13 @@ type LinuxBridgeClient struct { func NewLinuxBridgeClient( bridgeName string, hostInterfaceName string, - epInfo EndpointInfo, + nwInfo EndpointInfo, nl netlink.NetlinkInterface, plc platform.ExecClient, ) *LinuxBridgeClient { client := &LinuxBridgeClient{ bridgeName: bridgeName, - epInfo: epInfo, + nwInfo: nwInfo, hostInterfaceName: hostInterfaceName, netlink: nl, nuClient: networkutils.NewNetworkUtils(nl, plc), @@ -112,9 +112,9 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error { return err } - if client.epInfo.IPV6Mode != "" { + if client.nwInfo.IPV6Mode != "" { // for ipv6 node cidr set broute accept - if err := ebtables.SetBrouteAcceptByCidr(&client.epInfo.Subnets[1].Prefix, ebtables.IPV6, ebtables.Append, ebtables.Accept); err != nil { + if err := ebtables.SetBrouteAcceptByCidr(&client.nwInfo.Subnets[1].Prefix, ebtables.IPV6, ebtables.Append, ebtables.Accept); err != nil { return err } @@ -137,7 +137,7 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error { } // Enable VEPA for host policy enforcement if necessary. - if client.epInfo.Mode == opModeTunnel { + if client.nwInfo.Mode == opModeTunnel { logger.Info("Enabling VEPA mode for", zap.String("hostInterfaceName", client.hostInterfaceName)) if err := ebtables.SetVepaMode(client.bridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Append); err != nil { return err @@ -152,7 +152,7 @@ func (client *LinuxBridgeClient) DeleteL2Rules(extIf *externalInterface) { ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete) ebtables.SetArpReply(extIf.IPAddresses[0].IP, extIf.MacAddress, ebtables.Delete) ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete) - if client.epInfo.IPV6Mode != "" { + if client.nwInfo.IPV6Mode != "" { if len(extIf.IPAddresses) > 1 { ebtables.SetBrouteAcceptByCidr(extIf.IPAddresses[1], ebtables.IPV6, ebtables.Delete, ebtables.Accept) } @@ -180,7 +180,7 @@ func (client *LinuxBridgeClient) SetHairpinOnHostInterface(enable bool) error { } func (client *LinuxBridgeClient) setBrouteRedirect(action string) error { - if client.epInfo.ServiceCidrs != "" { + if client.nwInfo.ServiceCidrs != "" { if err := ebtables.SetBrouteAcceptByCidr(nil, ebtables.IPV4, ebtables.Append, ebtables.RedirectAccept); err != nil { return err } From 0badbf211a209de8b0139ad31e6edada479c3ae0 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 21 May 2024 16:00:07 -0700 Subject: [PATCH 077/102] separate endpoint and network policies in endpoint info behavior should not change except in hnsv1, where network policies passed into network create call will NOT include endpoint policies endpoint policies always include network policies --- cni/network/network.go | 9 +++++---- network/endpoint.go | 3 ++- network/endpoint_windows.go | 4 ++-- network/network_windows.go | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index ac4100d46c..5934e3c48f 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -711,7 +711,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, BridgeName: opt.ipamAddConfig.nwCfg.Bridge, NetworkDNS: nwDNSInfo, // nw and ep dns infos are separated to avoid possible conflicts - Policies: opt.policies, // CHECK: Should there be endpoint policies and a separate network policies? + NetworkPolicies: opt.policies, // nw and ep policies separated to avoid possible conflicts NetNs: opt.ipamAddConfig.args.Netns, Options: opt.ipamAddConfig.options, DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, @@ -743,7 +743,8 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn logger.Error("Failed to get endpoint policies", zap.Error(err)) return nil, err } - + // create endpoint policies by appending to network policies + // the value passed into NetworkPolicies should be unaffected since we reassign here opt.policies = append(opt.policies, endpointPolicies...) vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) @@ -777,7 +778,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn endpointInfo.IfName = opt.args.IfName endpointInfo.Data = make(map[string]interface{}) endpointInfo.EndpointDNS = epDNSInfo - endpointInfo.Policies = opt.policies + endpointInfo.EndpointPolicies = opt.policies endpointInfo.IPsToRouteViaHost = opt.nwCfg.IPsToRouteViaHost endpointInfo.EnableSnatOnHost = opt.nwCfg.EnableSnatOnHost endpointInfo.EnableMultiTenancy = opt.nwCfg.MultiTenancy @@ -804,7 +805,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn logger.Error("failed to get policies from runtime configurations", zap.Error(err)) return nil, plugin.Errorf(err.Error()) } - endpointInfo.Policies = append(endpointInfo.Policies, epPolicies...) + endpointInfo.EndpointPolicies = append(endpointInfo.EndpointPolicies, epPolicies...) if opt.ipamAddResult.ipv6Enabled { // not specific to this particular interface endpointInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working diff --git a/network/endpoint.go b/network/endpoint.go index 0087bdf8eb..6bc4486bc9 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -77,7 +77,8 @@ type EndpointInfo struct { IPsToRouteViaHost []string InfraVnetIP net.IPNet Routes []RouteInfo - Policies []policy.Policy + EndpointPolicies []policy.Policy // used in windows + NetworkPolicies []policy.Policy // used in windows Gateways []net.IP EnableSnatOnHost bool EnableInfraVnet bool diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 66f291dfe2..1a3d949d13 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -106,7 +106,7 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo, plc platform.ExecC VirtualNetwork: nw.HnsId, DNSSuffix: epInfo.EndpointDNS.Suffix, DNSServerList: strings.Join(epInfo.EndpointDNS.Servers, ","), - Policies: policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy), + Policies: policy.SerializePolicies(policy.EndpointPolicy, epInfo.EndpointPolicies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy), } // HNS currently supports one IP address and one IPv6 address per endpoint. @@ -235,7 +235,7 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE } hcnEndpoint.MacAddress = macAddress - if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.NATInfo); err == nil { + if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.EndpointPolicies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.NATInfo); err == nil { for _, epPolicy := range endpointPolicies { hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) } diff --git a/network/network_windows.go b/network/network_windows.go index 6d7feaf49d..8c8fd3c2d7 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -115,7 +115,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *EndpointInfo, extIf *exter Name: nwInfo.NetworkID, NetworkAdapterName: networkAdapterName, DNSServerList: strings.Join(nwInfo.NetworkDNS.Servers, ","), - Policies: policy.SerializePolicies(policy.NetworkPolicy, nwInfo.Policies, nil, false, false), + Policies: policy.SerializePolicies(policy.NetworkPolicy, nwInfo.NetworkPolicies, nil, false, false), } // Set the VLAN and OutboundNAT policies From 87f1bfe585e7a5b05cbd3eb6a2a7d62bbf4b4979 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 22 May 2024 00:38:56 -0700 Subject: [PATCH 078/102] address feedback from reviewers --- cni/network/invoker_cns.go | 2 +- cni/network/network.go | 32 ++++++++-------------- cni/network/network_windows.go | 6 ++-- network/endpoint.go | 4 +-- network/endpoint_windows.go | 7 ++--- network/manager.go | 9 ++++-- network/secondary_endpoint_client_linux.go | 2 +- 7 files changed, 27 insertions(+), 35 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 60c433aee7..831a3b6c93 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -195,7 +195,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro return IPAMAddResult{}, err } default: - logger.Info("Unknown NIC type received from cns pod ip info", zap.String("nicType", string(info.nicType))) + logger.Warn("Unknown NIC type received from cns pod ip info", zap.String("nicType", string(info.nicType))) } } diff --git a/cni/network/network.go b/cni/network/network.go index 5934e3c48f..6b2aa4e902 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -578,7 +578,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { epInfos := []*network.EndpointInfo{} infraSeen := false - i := 0 + endpointIndex := 0 for key := range ipamAddResult.interfaceInfo { ifInfo := ipamAddResult.interfaceInfo[key] @@ -598,13 +598,11 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { enableSnatForDNS: enableSnatForDNS, natInfo: natInfo, networkID: networkID, - - ifInfo: &ifInfo, - ipamAddConfig: &ipamAddConfig, - ipv6Enabled: ipamAddResult.ipv6Enabled, - - infraSeen: &infraSeen, - idx: i, + ifInfo: &ifInfo, + ipamAddConfig: &ipamAddConfig, + ipv6Enabled: ipamAddResult.ipv6Enabled, + infraSeen: &infraSeen, + endpointIndex: endpointIndex, } var epInfo *network.EndpointInfo epInfo, err = plugin.createEpInfo(&createEpInfoOpt) @@ -615,7 +613,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // TODO: should this statement be based on the current iteration instead of the constant ifIndex? // TODO figure out where to put telemetry: sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", // ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) - i++ + endpointIndex++ } cnsclient, err := cnscli.New(nwCfg.CNSUrl, defaultRequestTimeout) if err != nil { @@ -683,8 +681,8 @@ type createEpInfoOpt struct { ipamAddConfig *IPAMAddConfig ipv6Enabled bool - infraSeen *bool // Only the first infra gets args.ifName, even if the second infra is on a different network - idx int + infraSeen *bool // Only the first infra gets args.ifName, even if the second infra is on a different network + endpointIndex int } func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointInfo, error) { // you can modify to pass in whatever else you need @@ -769,7 +767,7 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn endpointID = plugin.nm.GetEndpointID(opt.args.ContainerID, opt.args.IfName) *opt.infraSeen = true } else { - endpointID = plugin.nm.GetEndpointID(opt.args.ContainerID, strconv.Itoa(opt.idx)) + endpointID = plugin.nm.GetEndpointID(opt.args.ContainerID, strconv.Itoa(opt.endpointIndex)) } endpointInfo.EndpointID = endpointID @@ -1079,6 +1077,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { } // Initialize values from network config. if err != nil { + // if swift v1 multitenancy and we got an error retrieving the nwInfo // If error is not found error, then we ignore it, to comply with CNI SPEC. if network.IsNetworkNotFoundError(err) { err = nil @@ -1462,12 +1461,3 @@ func convertCniResultToInterfaceInfo(result *cniTypesCurr.Result) network.Interf return interfaceInfo } - -// func findInfraNicInterfaceKey(ifInfos map[string]network.InterfaceInfo) network.InterfaceInfo { -// for _, ifInfo := range ifInfos { -// if ifInfo.NICType == cns.InfraNIC { -// return &ifInfo -// } -// } -// return nil -// } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 6208397eb0..34298c155e 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -29,8 +29,8 @@ import ( var ( snatConfigFileName = filepath.FromSlash(os.Getenv("TEMP")) + "\\snatConfig" // windows build for version 1903 - win1903Version = 18362 - ipv6SubnetInfoCount = 2 + win1903Version = 18362 + dualStackCount = 2 ) /* handleConsecutiveAdd handles consecutive add calls for infrastructure containers on Windows platform. @@ -377,7 +377,7 @@ func getLoopbackDSRPolicy(args PolicyArgs) ([]policy.Policy, error) { func getIPV6EndpointPolicy(subnetInfos []network.SubnetInfo) (policy.Policy, error) { var eppolicy policy.Policy - if len(subnetInfos) < ipv6SubnetInfoCount { + if len(subnetInfos) < dualStackCount { return eppolicy, fmt.Errorf("network state doesn't have ipv6 subnet") } diff --git a/network/endpoint.go b/network/endpoint.go index 6bc4486bc9..5c0f1d6893 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -139,8 +139,8 @@ type InterfaceInfo struct { DNS DNSInfo NICType cns.NICType SkipDefaultRoutes bool - HostSubnetPrefix net.IPNet // Move this field from ipamAddResult - NCResponse *cns.GetNetworkContainerResponse // TODO: create a new struct and keep fields that are being used + HostSubnetPrefix net.IPNet // Move this field from ipamAddResult + NCResponse *cns.GetNetworkContainerResponse } type IPConfig struct { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 1a3d949d13..c5e6b90f19 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -394,7 +394,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( } nicName := epInfo.IfName - // infra nic nicname will look like eth0, but delegated/secondary nics will look like "vEthernet 2" + // infra nic nicname will look like eth0, but delegated/secondary nics will look like "vEthernet x" where x is 1-7 if epInfo.NICType != cns.InfraNIC { nicName = epInfo.MasterIfName } @@ -417,9 +417,8 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( ContainerID: epInfo.ContainerID, PODName: epInfo.PODName, PODNameSpace: epInfo.PODNameSpace, - // SecondaryInterfaces: make(map[string]*InterfaceInfo), - NICType: epInfo.NICType, - NICName: nicName, + NICType: epInfo.NICType, + NICName: nicName, } for _, route := range epInfo.Routes { diff --git a/network/manager.go b/network/manager.go index e7f418128e..2f82a38227 100644 --- a/network/manager.go +++ b/network/manager.go @@ -39,7 +39,7 @@ const ( ContainerIDLength = 8 EndpointIfIndex = 0 // Azure CNI supports only one interface DefaultNetworkID = "azure" - dummyGUID = "12345678-1234-1234-1234-123456789012" + dummyGUID = "12345678-1234-1234-1234-123456789012" // guid to trigger hnsv2 in windows ) var Ipv4DefaultRouteDstPrefix = net.IPNet{ @@ -479,8 +479,11 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E } func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *EndpointInfo) error { + // we want to always use hnsv2 in stateless + // hnsv2 is only enabled if NetNs has a valid guid and the hnsv2 api is supported + // by passing in a dummy guid, we satisfy the first condition nw := &network{ - Id: networkID, // CHECK: currently unused in stateless cni + Id: networkID, // currently unused in stateless cni HnsId: epInfo.HNSNetworkID, Mode: opModeTransparentVlan, SnatBridgeIP: "", @@ -505,7 +508,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint NetworkContainerID: epInfo.NetworkContainerID, // we don't use this as long as AllowInboundFromHostToNC and AllowInboundFromNCToHost are false NetNs: dummyGUID, // to trigger hnsv2, windows NICType: epInfo.NICType, - NICName: epInfo.IfName, + NICName: epInfo.IfName, // TODO: For stateless cni linux populate NICName or IfName here to use in deletion in secondary endpoint client } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) err := nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index efa7a20131..962aae5ed7 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -169,7 +169,7 @@ func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { logger.Error("Failed to exit netns with", zap.Error(newErrorSecondaryEndpointClient(err))) } }() - + // TODO: For stateless cni linux, check if delegated vmnic type, and if so, delete using the *endpoint*'s ifname or nicname for iface := range ep.SecondaryInterfaces { if err := client.netlink.SetLinkNetNs(iface, uintptr(vmns)); err != nil { logger.Error("Failed to move interface", zap.String("IfName", iface), zap.Error(newErrorSecondaryEndpointClient(err))) From 139a3a1a0036d18a0c037774d9895bc3c264e19c Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 22 May 2024 13:54:56 -0700 Subject: [PATCH 079/102] address feedback and account for case where cns provides info without nic type if nic type is empty from cns in invoker cns, we assume it is infra nic type and populate it with infra nic type --- cni/network/invoker_cns.go | 13 ++++++------- cni/network/network.go | 10 ++++------ network/endpoint.go | 2 -- network/endpoint_linux.go | 2 +- network/manager.go | 3 ++- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 831a3b6c93..450a25bc8a 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -181,6 +181,11 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro return IPAMAddResult{}, err } case cns.InfraNIC: + fallthrough + case "": + // if we change from legacy cns, the nicType will be empty, so we assume it is infra nic + info.nicType = cns.InfraNIC + // only count dualstack interface once _, exist := addResult.interfaceInfo[key] if !exist { @@ -370,7 +375,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } ip, ncIPNet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix)) - if ip == nil { + if ip == nil || err != nil { return errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") } @@ -393,12 +398,6 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } } - if err != nil { - logger.Error("Error finding InfraNIC interface", - zap.Error(err)) - return errors.Wrap(err, "error finding InfraNIC interface") - } - // get the name of the primary IP address _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) if err != nil { diff --git a/cni/network/network.go b/cni/network/network.go index 6b2aa4e902..a1f8a687d4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -562,9 +562,8 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { defer func() { //nolint:gocritic if err != nil { - // for multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips - // if nwCfg.Multitenancy is false or ipam type is not azure cns - if !(nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS) { + // for swift v1 multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips + if !nwCfg.MultiTenancy { for _, ifInfo := range ipamAddResult.interfaceInfo { // This used to only be called for infraNIC, test if this breaks scenarios // If it does then will have to search for infraNIC @@ -640,7 +639,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } } }() - // CHECK: why does the cns client deal with apipa?-- "cns client" only used in windows to create apipa + err = plugin.nm.EndpointCreate(cnsclient, epInfos) if err != nil { return errors.Wrap(err, "failed to create endpoint") // behavior can change if you don't assign to err prior to returning @@ -657,7 +656,7 @@ func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string { case cns.DelegatedVMNIC: return plugin.findInterfaceByMAC(opt.ifInfo.MacAddress.String()) case cns.BackendNIC: - return "" + fallthrough default: return "" } @@ -713,7 +712,6 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn NetNs: opt.ipamAddConfig.args.Netns, Options: opt.ipamAddConfig.options, DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - IPAMType: opt.ipamAddConfig.nwCfg.IPAM.Type, // present infra only IsIPv6Enabled: opt.ipv6Enabled, // present infra only } diff --git a/network/endpoint.go b/network/endpoint.go index 5c0f1d6893..d8e43f33fe 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -107,12 +107,10 @@ type EndpointInfo struct { NetworkID string Mode string Subnets []SubnetInfo - PodSubnet SubnetInfo BridgeName string NetNs string // used in windows Options map[string]interface{} DisableHairpinOnHostInterface bool - IPAMType string IsIPv6Enabled bool HostSubnetPrefix string // can be used later to add an external interface diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 73da43ca79..ce04104d2d 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -99,7 +99,7 @@ func (nw *network) newEndpointImpl( } nicName := epInfo.IfName - // infra nic nicname will look like eth0, but delegated/secondary nics will look like "eth1 eth2 eth3" + // infra nic nicname will look like eth0, and delegated/secondary nics will be moved into the container namespace if epInfo.NICType != cns.InfraNIC { nicName = epInfo.MasterIfName } diff --git a/network/manager.go b/network/manager.go index 2f82a38227..9e5e45b8c9 100644 --- a/network/manager.go +++ b/network/manager.go @@ -39,7 +39,8 @@ const ( ContainerIDLength = 8 EndpointIfIndex = 0 // Azure CNI supports only one interface DefaultNetworkID = "azure" - dummyGUID = "12345678-1234-1234-1234-123456789012" // guid to trigger hnsv2 in windows + // TODO: Remove dummy GUID and come up with more permanent solution + dummyGUID = "12345678-1234-1234-1234-123456789012" // guid to trigger hnsv2 in windows ) var Ipv4DefaultRouteDstPrefix = net.IPNet{ From b13e84fcff7a73020bea580b45a707043bb27e5a Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 22 May 2024 20:00:35 -0700 Subject: [PATCH 080/102] address feedback to declare endpoint info once and populate all fields at once moved add subnets to after endpoint info created moved retrieval of all endpoint policies (from getEndpointPolicies and getPoliciesFromRuntimeCfg)until after endpoint info created network policies are just passed in from the args unaltered --- cni/network/network.go | 129 +++++++++++++++++---------------- cni/network/network_linux.go | 1 + cni/network/network_windows.go | 1 + 3 files changed, 68 insertions(+), 63 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index a1f8a687d4..6b84a4649b 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -685,7 +685,6 @@ type createEpInfoOpt struct { } func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointInfo, error) { // you can modify to pass in whatever else you need - var endpointInfo network.EndpointInfo // ensure we can find the master interface opt.ifInfo.HostSubnetPrefix.IP = opt.ifInfo.HostSubnetPrefix.IP.Mask(opt.ifInfo.HostSubnetPrefix.Mask) opt.ipamAddConfig.nwCfg.IPAM.Subnet = opt.ifInfo.HostSubnetPrefix.String() @@ -700,26 +699,8 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn if err != nil { return nil, err } - // this struct is for organization and represents fields from the network to be merged into the endpoint info later - endpointInfo = network.EndpointInfo{ - NetworkID: opt.networkID, - Mode: opt.ipamAddConfig.nwCfg.Mode, - MasterIfName: masterIfName, - AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, - BridgeName: opt.ipamAddConfig.nwCfg.Bridge, - NetworkDNS: nwDNSInfo, // nw and ep dns infos are separated to avoid possible conflicts - NetworkPolicies: opt.policies, // nw and ep policies separated to avoid possible conflicts - NetNs: opt.ipamAddConfig.args.Netns, - Options: opt.ipamAddConfig.options, - DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, - IsIPv6Enabled: opt.ipv6Enabled, // present infra only - } - if err = addSubnetToEndpointInfo(*opt.ifInfo, &endpointInfo); err != nil { - logger.Info("Failed to add subnets to endpointInfo", zap.Error(err)) - return nil, err - } - setNetworkOptions(opt.ifInfo.NCResponse, &endpointInfo) + networkPolicies := opt.policies // save network policies before we modify the slice pointer for ep policies // populate endpoint info epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, opt.ifInfo.DNS, opt.k8sNamespace) // Probably won't panic if given bad values @@ -728,27 +709,13 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) return nil, err } - policyArgs := PolicyArgs{ - subnetInfos: endpointInfo.Subnets, // getEndpointPolicies requires nwInfo.Subnets only (checked) - nwCfg: opt.nwCfg, - ipconfigs: opt.ifInfo.IPConfigs, - } - endpointPolicies, err := getEndpointPolicies(policyArgs) - if err != nil { - - logger.Error("Failed to get endpoint policies", zap.Error(err)) - return nil, err - } - // create endpoint policies by appending to network policies - // the value passed into NetworkPolicies should be unaffected since we reassign here - opt.policies = append(opt.policies, endpointPolicies...) vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) if opt.nwCfg.Mode != OpModeTransparent { // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. // IT will result in unpredictable behavior if API server decides to // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", endpointInfo.NetworkID, opt.args.ContainerID, opt.args.IfName) + vethName = fmt.Sprintf("%s%s%s", opt.networkID, opt.args.ContainerID, opt.args.IfName) } // for secondary (Populate addresses) @@ -768,34 +735,70 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn endpointID = plugin.nm.GetEndpointID(opt.args.ContainerID, strconv.Itoa(opt.endpointIndex)) } - endpointInfo.EndpointID = endpointID - endpointInfo.ContainerID = opt.args.ContainerID - endpointInfo.NetNsPath = opt.args.Netns // probably same value as epInfo.NetNs - endpointInfo.IfName = opt.args.IfName - endpointInfo.Data = make(map[string]interface{}) - endpointInfo.EndpointDNS = epDNSInfo - endpointInfo.EndpointPolicies = opt.policies - endpointInfo.IPsToRouteViaHost = opt.nwCfg.IPsToRouteViaHost - endpointInfo.EnableSnatOnHost = opt.nwCfg.EnableSnatOnHost - endpointInfo.EnableMultiTenancy = opt.nwCfg.MultiTenancy - endpointInfo.EnableInfraVnet = opt.enableInfraVnet - endpointInfo.EnableSnatForDns = opt.enableSnatForDNS - endpointInfo.PODName = opt.k8sPodName - endpointInfo.PODNameSpace = opt.k8sNamespace - endpointInfo.SkipHotAttachEp = false // Hot attach at the time of endpoint creation - endpointInfo.IPV6Mode = opt.nwCfg.IPV6Mode - endpointInfo.VnetCidrs = opt.nwCfg.VnetCidrs - endpointInfo.ServiceCidrs = opt.nwCfg.ServiceCidrs - endpointInfo.NATInfo = opt.natInfo - endpointInfo.NICType = opt.ifInfo.NICType - endpointInfo.SkipDefaultRoutes = opt.ifInfo.SkipDefaultRoutes - endpointInfo.Routes = opt.ifInfo.Routes - // added the following for delegated vm nic - endpointInfo.IPAddresses = addresses - endpointInfo.MacAddress = opt.ifInfo.MacAddress - // the following is used for creating an external interface if we can't find an existing network - endpointInfo.HostSubnetPrefix = opt.ifInfo.HostSubnetPrefix.String() + endpointInfo := network.EndpointInfo{ + NetworkID: opt.networkID, + Mode: opt.ipamAddConfig.nwCfg.Mode, + MasterIfName: masterIfName, + AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, + BridgeName: opt.ipamAddConfig.nwCfg.Bridge, + NetworkDNS: nwDNSInfo, // nw and ep dns infos are separated to avoid possible conflicts + NetworkPolicies: networkPolicies, // nw and ep policies separated to avoid possible conflicts + NetNs: opt.ipamAddConfig.args.Netns, + Options: opt.ipamAddConfig.options, + DisableHairpinOnHostInterface: opt.ipamAddConfig.nwCfg.DisableHairpinOnHostInterface, + IsIPv6Enabled: opt.ipv6Enabled, // present infra only + + EndpointID: endpointID, + ContainerID: opt.args.ContainerID, + NetNsPath: opt.args.Netns, // probably same value as epInfo.NetNs + IfName: opt.args.IfName, + Data: make(map[string]interface{}), + EndpointDNS: epDNSInfo, + // endpoint policies are populated later + IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, + EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, + EnableMultiTenancy: opt.nwCfg.MultiTenancy, + EnableInfraVnet: opt.enableInfraVnet, + EnableSnatForDns: opt.enableSnatForDNS, + PODName: opt.k8sPodName, + PODNameSpace: opt.k8sNamespace, + SkipHotAttachEp: false, // Hot attach at the time of endpoint creation + IPV6Mode: opt.nwCfg.IPV6Mode, + VnetCidrs: opt.nwCfg.VnetCidrs, + ServiceCidrs: opt.nwCfg.ServiceCidrs, + NATInfo: opt.natInfo, + NICType: opt.ifInfo.NICType, + SkipDefaultRoutes: opt.ifInfo.SkipDefaultRoutes, + Routes: opt.ifInfo.Routes, + // added the following for delegated vm nic + IPAddresses: addresses, + MacAddress: opt.ifInfo.MacAddress, + // the following is used for creating an external interface if we can't find an existing network + HostSubnetPrefix: opt.ifInfo.HostSubnetPrefix.String(), + } + + if err = addSubnetToEndpointInfo(*opt.ifInfo, &endpointInfo); err != nil { + logger.Info("Failed to add subnets to endpointInfo", zap.Error(err)) + return nil, err + } + setNetworkOptions(opt.ifInfo.NCResponse, &endpointInfo) + // update endpoint policies + policyArgs := PolicyArgs{ + subnetInfos: endpointInfo.Subnets, // getEndpointPolicies requires nwInfo.Subnets only (checked) + nwCfg: opt.nwCfg, + ipconfigs: opt.ifInfo.IPConfigs, + } + endpointPolicies, err := getEndpointPolicies(policyArgs) + if err != nil { + logger.Error("Failed to get endpoint policies", zap.Error(err)) + return nil, err + } + // create endpoint policies by appending to network policies + // the value passed into NetworkPolicies should be unaffected since we reassign here + opt.policies = append(opt.policies, endpointPolicies...) + endpointInfo.EndpointPolicies = opt.policies + // add even more endpoint policies epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) // not specific to delegated or infra if err != nil { logger.Error("failed to get policies from runtime configurations", zap.Error(err)) @@ -850,7 +853,7 @@ func (plugin *NetPlugin) getNetworkDNSSettings(nwCfg *cni.NetworkConfig, dns net return nwDNSInfo, nil } -// construct network info with ipv4/ipv6 subnets +// construct network info with ipv4/ipv6 subnets (updates subnets field) func addSubnetToEndpointInfo(interfaceInfo network.InterfaceInfo, nwInfo *network.EndpointInfo) error { for _, ipConfig := range interfaceInfo.IPConfigs { ip, podSubnetPrefix, err := net.ParseCIDR(ipConfig.Address.String()) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 68702ff17f..eef2c49243 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -41,6 +41,7 @@ func addSnatForDNS(gwIPString string, epInfo *network.EndpointInfo, result *netw result.Routes = append(result.Routes, network.RouteInfo{Dst: *dnsIPNet, Gw: gwIP}) } +// updates options field func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.EndpointInfo) { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { logger.Info("Setting Network Options") diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 34298c155e..5665e85d1f 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -117,6 +117,7 @@ func addDefaultRoute(_ string, _ *network.EndpointInfo, _ *network.InterfaceInfo func addSnatForDNS(_ string, _ *network.EndpointInfo, _ *network.InterfaceInfo) { } +// updates options field func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.EndpointInfo) { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { logger.Info("Setting Network Options") From a1af26a07ae746be1d2272d5dd4e34598f46219c Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 22 May 2024 21:47:49 -0700 Subject: [PATCH 081/102] use ifname instead of nicname field in endpoint struct as key in stateless --- network/endpoint.go | 2 -- network/endpoint_linux.go | 11 ++++++----- network/endpoint_windows.go | 4 +--- network/manager.go | 6 +++--- network/secondary_endpoint_client_linux.go | 2 +- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index d8e43f33fe..bea1090b2f 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -58,8 +58,6 @@ type endpoint struct { SecondaryInterfaces map[string]*InterfaceInfo // Store nic type since we no longer populate SecondaryInterfaces NICType cns.NICType - // Store the key used in stateless - NICName string } // EndpointInfo contains read-only information about an endpoint. diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index ce04104d2d..04b4191e72 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -90,6 +90,8 @@ func (nw *network) newEndpointImpl( logger.Info("Generate veth name based on the key provided", zap.String("key", key)) vethname := generateVethName(key) hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) + // we don't save the contIfName here or below because it will be renamed to ep.IfName anyway when we call SetupContainerInterfaces in the clients + // however, contIfName is still passed into our clients contIfName = fmt.Sprintf("%s%s2", hostVEthInterfacePrefix, vethname) } else { // Create a veth pair. @@ -99,14 +101,15 @@ func (nw *network) newEndpointImpl( } nicName := epInfo.IfName - // infra nic nicname will look like eth0, and delegated/secondary nics will be moved into the container namespace + // infra nic nicname will look like eth0, and delegated/secondary nics will be moved into the container namespace if epInfo.NICType != cns.InfraNIC { nicName = epInfo.MasterIfName } ep := &endpoint{ - Id: defaultEpInfo.EndpointID, - IfName: contIfName, // container veth pair name. In cnm, we won't rename this and docker expects veth name. + Id: defaultEpInfo.EndpointID, + // IfName should end up being eth0 in non-delegated nic cases + IfName: nicName, // container veth pair name. In cnm, we won't rename this and docker expects veth name. HostIfName: hostIfName, InfraVnetIP: defaultEpInfo.InfraVnetIP, LocalIP: localIP, @@ -125,8 +128,6 @@ func (nw *network) newEndpointImpl( Routes: defaultEpInfo.Routes, SecondaryInterfaces: make(map[string]*InterfaceInfo), NICType: defaultEpInfo.NICType, - // Should end up being eth0 in non-delegated nic cases - NICName: nicName, // CHECK: Should find interface name by mac address for linux secondaries } if nw.extIf != nil { ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index c5e6b90f19..27931908a1 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -171,7 +171,6 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo, plc platform.ExecC NetNs: epInfo.NetNsPath, ContainerID: epInfo.ContainerID, NICType: epInfo.NICType, - NICName: epInfo.IfName, } for _, route := range epInfo.Routes { @@ -404,7 +403,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( Id: hcnEndpoint.Name, HnsId: hnsResponse.Id, SandboxKey: epInfo.ContainerID, - IfName: epInfo.IfName, + IfName: nicName, IPAddresses: epInfo.IPAddresses, Gateways: []net.IP{gateway}, DNS: epInfo.EndpointDNS, @@ -418,7 +417,6 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( PODName: epInfo.PODName, PODNameSpace: epInfo.PODNameSpace, NICType: epInfo.NICType, - NICName: nicName, } for _, route := range epInfo.Routes { diff --git a/network/manager.go b/network/manager.go index 9e5e45b8c9..cd28c28360 100644 --- a/network/manager.go +++ b/network/manager.go @@ -509,7 +509,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint NetworkContainerID: epInfo.NetworkContainerID, // we don't use this as long as AllowInboundFromHostToNC and AllowInboundFromNCToHost are false NetNs: dummyGUID, // to trigger hnsv2, windows NICType: epInfo.NICType, - NICName: epInfo.IfName, // TODO: For stateless cni linux populate NICName or IfName here to use in deletion in secondary endpoint client + IfName: epInfo.IfName, // TODO: For stateless cni linux populate IfName here to use in deletion in secondary endpoint client } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) err := nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) @@ -781,7 +781,7 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI // filling out the InfraNIC from the state epInfo.IPAddresses = ipInfo.IPv4 epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...) - epInfo.IfName = ifName // epInfo.IfName is set to the value of the NICName when the endpoint was added + epInfo.IfName = ifName // epInfo.IfName is set to the value of ep.IfName when the endpoint was added // sidenote: ifname doesn't seem to be used in linux (or even windows) deletion epInfo.HostIfName = ipInfo.HostVethName epInfo.HNSEndpointID = ipInfo.HnsEndpointID @@ -813,7 +813,7 @@ func generateCNSIPInfoMap(eps []*endpoint) map[string]*restserver.IPInfo { ifNametoIPInfoMap := make(map[string]*restserver.IPInfo) // key : interface name, value : IPInfo for _, ep := range eps { - ifNametoIPInfoMap[ep.NICName] = &restserver.IPInfo{ // in windows, the nicname is args ifname, in linux, it's ethX + ifNametoIPInfoMap[ep.IfName] = &restserver.IPInfo{ // in windows, the nicname is args ifname, in linux, it's ethX NICType: ep.NICType, HnsEndpointID: ep.HnsId, HnsNetworkID: ep.HNSNetworkID, diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index 962aae5ed7..46fc3c26f4 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -169,7 +169,7 @@ func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { logger.Error("Failed to exit netns with", zap.Error(newErrorSecondaryEndpointClient(err))) } }() - // TODO: For stateless cni linux, check if delegated vmnic type, and if so, delete using the *endpoint*'s ifname or nicname + // TODO: For stateless cni linux, check if delegated vmnic type, and if so, delete using this *endpoint* struct's ifname for iface := range ep.SecondaryInterfaces { if err := client.netlink.SetLinkNetNs(iface, uintptr(vmns)); err != nil { logger.Error("Failed to move interface", zap.String("IfName", iface), zap.Error(newErrorSecondaryEndpointClient(err))) From 0a9c2724365b2e44026ff078ec8662c8813f4106 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 23 May 2024 12:55:03 -0400 Subject: [PATCH 082/102] convert macaddress only nictype is delegatedvmnic --- network/endpoint_windows.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 27931908a1..30d350e096 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -227,8 +227,9 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE } // macAddress type for InfraNIC is like "60:45:bd:12:45:65" + // if NICType is delegatedVMNIC, convert the macaddress format macAddress := epInfo.MacAddress.String() - if epInfo.NICType != cns.InfraNIC { + if epInfo.NICType == cns.DelegatedVMNIC { // convert the format of macAddress that HNS can accept, i.e, "60-45-bd-12-45-65" if NIC type is delegated NIC macAddress = strings.Join(strings.Split(macAddress, ":"), "-") } From 2c177f9c8ad72d722d841188d2c60e0d60512783 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 23 May 2024 11:13:07 -0700 Subject: [PATCH 083/102] address feedback by removing network dns settings --- cni/network/network.go | 16 ---------------- network/endpoint.go | 1 - network/manager.go | 1 - network/network_linux.go | 1 - network/network_windows.go | 5 ----- 5 files changed, 24 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 6b84a4649b..06e56033e2 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -695,11 +695,6 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn return nil, err } - nwDNSInfo, err := plugin.getNetworkDNSSettings(opt.ipamAddConfig.nwCfg, opt.ifInfo.DNS) - if err != nil { - return nil, err - } - networkPolicies := opt.policies // save network policies before we modify the slice pointer for ep policies // populate endpoint info @@ -741,7 +736,6 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn MasterIfName: masterIfName, AdapterName: opt.ipamAddConfig.nwCfg.AdapterName, BridgeName: opt.ipamAddConfig.nwCfg.Bridge, - NetworkDNS: nwDNSInfo, // nw and ep dns infos are separated to avoid possible conflicts NetworkPolicies: networkPolicies, // nw and ep policies separated to avoid possible conflicts NetNs: opt.ipamAddConfig.args.Netns, Options: opt.ipamAddConfig.options, @@ -843,16 +837,6 @@ func (plugin *NetPlugin) cleanupAllocationOnError( } } -func (plugin *NetPlugin) getNetworkDNSSettings(nwCfg *cni.NetworkConfig, dns network.DNSInfo) (network.DNSInfo, error) { - nwDNSInfo, err := getNetworkDNSSettings(nwCfg, dns) - if err != nil { - err = plugin.Errorf("Failed to getDNSSettings: %v", err) - return network.DNSInfo{}, err - } - logger.Info("DNS Info", zap.Any("info", nwDNSInfo)) - return nwDNSInfo, nil -} - // construct network info with ipv4/ipv6 subnets (updates subnets field) func addSubnetToEndpointInfo(interfaceInfo network.InterfaceInfo, nwInfo *network.EndpointInfo) error { for _, ipConfig := range interfaceInfo.IPConfigs { diff --git a/network/endpoint.go b/network/endpoint.go index bea1090b2f..a5c36fa854 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -70,7 +70,6 @@ type EndpointInfo struct { IfIndex int MacAddress net.HardwareAddr EndpointDNS DNSInfo - NetworkDNS DNSInfo IPAddresses []net.IPNet IPsToRouteViaHost []string InfraVnetIP net.IPNet diff --git a/network/manager.go b/network/manager.go index cd28c28360..0ac288a4ba 100644 --- a/network/manager.go +++ b/network/manager.go @@ -358,7 +358,6 @@ func (nm *networkManager) GetNetworkInfo(networkID string) (EndpointInfo, error) Subnets: nw.Subnets, Mode: nw.Mode, EnableSnatOnHost: nw.EnableSnatOnHost, - NetworkDNS: nw.DNS, Options: make(map[string]interface{}), } diff --git a/network/network_linux.go b/network/network_linux.go index a8019de06e..7f688d2b22 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -121,7 +121,6 @@ func (nm *networkManager) newNetworkImpl(nwInfo *EndpointInfo, extIf *externalIn Endpoints: make(map[string]*endpoint), extIf: extIf, VlanId: vlanid, - DNS: nwInfo.NetworkDNS, EnableSnatOnHost: nwInfo.EnableSnatOnHost, } diff --git a/network/network_windows.go b/network/network_windows.go index 8c8fd3c2d7..e588bdc04d 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -114,7 +114,6 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *EndpointInfo, extIf *exter hnsNetwork := &hcsshim.HNSNetwork{ Name: nwInfo.NetworkID, NetworkAdapterName: networkAdapterName, - DNSServerList: strings.Join(nwInfo.NetworkDNS.Servers, ","), Policies: policy.SerializePolicies(policy.NetworkPolicy, nwInfo.NetworkPolicies, nil, false, false), } @@ -234,10 +233,6 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *EndpointInfo, extIf *exter // Initialize HNS network. hcnNetwork := &hcn.HostComputeNetwork{ Name: nwInfo.NetworkID, - Dns: hcn.Dns{ - Domain: nwInfo.NetworkDNS.Suffix, - ServerList: nwInfo.NetworkDNS.Servers, - }, Ipams: []hcn.Ipam{ { Type: hcnIpamTypeStatic, From b1242bd98d31e7dc92f660291a3813482f5d4c86 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 24 May 2024 10:05:39 -0700 Subject: [PATCH 084/102] address linter issues (noop) --- cni/network/invoker_cns.go | 4 +--- cni/network/network.go | 2 -- network/endpoint_windows.go | 4 ++-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 450a25bc8a..54c5bca301 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -180,9 +180,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig, key); err != nil { return IPAMAddResult{}, err } - case cns.InfraNIC: - fallthrough - case "": + case cns.InfraNIC, "": // if we change from legacy cns, the nicType will be empty, so we assume it is infra nic info.nicType = cns.InfraNIC diff --git a/cni/network/network.go b/cni/network/network.go index 06e56033e2..7e48de1042 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -655,8 +655,6 @@ func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string { return plugin.findMasterInterfaceBySubnet(opt.ipamAddConfig.nwCfg, &opt.ifInfo.HostSubnetPrefix) case cns.DelegatedVMNIC: return plugin.findInterfaceByMAC(opt.ifInfo.MacAddress.String()) - case cns.BackendNIC: - fallthrough default: return "" } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 30d350e096..be226961f7 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -235,8 +235,8 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE } hcnEndpoint.MacAddress = macAddress - if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.EndpointPolicies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.NATInfo); err == nil { - for _, epPolicy := range endpointPolicies { + if epPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.EndpointPolicies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.NATInfo); err == nil { + for _, epPolicy := range epPolicies { hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) } } else { From 9ee7b91be90a5536b81251e5004d45aa4ae0a445 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 24 May 2024 13:54:22 -0700 Subject: [PATCH 085/102] address feedback and linter (noop) --- cni/network/multitenancy.go | 2 +- cni/network/multitenancy_mock.go | 2 +- cni/network/network.go | 2 ++ network/endpoint_windows.go | 4 +--- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 190865795b..67013863bd 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -227,7 +227,7 @@ func (m *Multitenancy) GetAllNetworkContainers( } ipconfig, routes := convertToIPConfigAndRouteInfo(ifInfo.NCResponse) - ifInfo.IPConfigs = []*network.IPConfig{ipconfig} + ifInfo.IPConfigs = append(ifInfo.IPConfigs, ipconfig) ifInfo.Routes = routes ifInfo.NICType = cns.InfraNIC diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index de6c7b8eb5..9c63279091 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -166,7 +166,7 @@ func (m *MockMultitenancy) GetAllNetworkContainers( } ipconfig, routes := convertToIPConfigAndRouteInfo(ifInfo.NCResponse) - ifInfo.IPConfigs = []*network.IPConfig{ipconfig} + ifInfo.IPConfigs = append(ifInfo.IPConfigs, ipconfig) ifInfo.Routes = routes ifInfo.NICType = cns.InfraNIC diff --git a/cni/network/network.go b/cni/network/network.go index 7e48de1042..2e3ee2bcb7 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -655,6 +655,8 @@ func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string { return plugin.findMasterInterfaceBySubnet(opt.ipamAddConfig.nwCfg, &opt.ifInfo.HostSubnetPrefix) case cns.DelegatedVMNIC: return plugin.findInterfaceByMAC(opt.ifInfo.MacAddress.String()) + case cns.BackendNIC: + return "" default: return "" } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index be226961f7..baf3d72671 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -236,9 +236,7 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE hcnEndpoint.MacAddress = macAddress if epPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.EndpointPolicies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.NATInfo); err == nil { - for _, epPolicy := range epPolicies { - hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) - } + hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicies...) } else { logger.Error("Failed to get endpoint policies due to", zap.Error(err)) return nil, err From 8fed0ec081533987eda1e9bf8e49da31cd3ca112 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 24 May 2024 14:28:03 -0700 Subject: [PATCH 086/102] remove unused consecutive add funcs (noop) --- cni/network/network_linux.go | 7 --- cni/network/network_windows.go | 80 ---------------------------------- network/endpoint.go | 2 +- 3 files changed, 1 insertion(+), 88 deletions(-) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index eef2c49243..ae1d0f234d 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -19,13 +19,6 @@ const ( const snatConfigFileName = "/tmp/snatConfig" -// handleConsecutiveAdd is a dummy function for Linux platform. -func (plugin *NetPlugin) handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointID string, networkID string, - nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig, -) (*cniTypesCurr.Result, error) { - return nil, nil -} - func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *network.InterfaceInfo) { _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 5665e85d1f..417bbd8532 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -18,8 +18,6 @@ import ( "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" hnsv2 "github.com/Microsoft/hcsshim/hcn" - cniSkel "github.com/containernetworking/cni/pkg/skel" - cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/100" "github.com/pkg/errors" "go.uber.org/zap" @@ -33,84 +31,6 @@ var ( dualStackCount = 2 ) -/* handleConsecutiveAdd handles consecutive add calls for infrastructure containers on Windows platform. - * This is a temporary work around for issue #57253 of Kubernetes. - * We can delete this if statement once they fix it. - * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 - */ -func (plugin *NetPlugin) handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, networkId string, - nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig, -) (*cniTypesCurr.Result, error) { - epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId) - if epInfo == nil { - return nil, nil - } - - // Return in case of HNSv2 as consecutive add call doesn't need to be handled - if useHnsV2, err := network.UseHnsV2(args.Netns); useHnsV2 { - return nil, err - } - - hnsEndpoint, err := network.Hnsv1.GetHNSEndpointByName(endpointId) - if hnsEndpoint != nil { - logger.Info("Found existing endpoint through hcsshim", - zap.Any("endpoint", hnsEndpoint)) - endpoint, _ := network.Hnsv1.GetHNSEndpointByID(hnsEndpoint.Id) - isAttached, _ := network.Hnsv1.IsAttached(endpoint, args.ContainerID) - // Attach endpoint if it's not attached yet. - if !isAttached { - logger.Info("Attaching endpoint to container", - zap.String("endpoint", hnsEndpoint.Id), - zap.String("container", args.ContainerID)) - err := network.Hnsv1.HotAttachEndpoint(args.ContainerID, hnsEndpoint.Id) - if err != nil { - logger.Error("Failed to hot attach shared endpoint to container", - zap.String("endpoint", hnsEndpoint.Id), - zap.String("container", args.ContainerID), - zap.Error(err)) - return nil, err - } - } - - // Populate result. - address := nwInfo.Subnets[0].Prefix - address.IP = hnsEndpoint.IPAddress - result := &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: address, - Gateway: net.ParseIP(hnsEndpoint.GatewayAddress), - }, - }, - Routes: []*cniTypes.Route{ - { - Dst: net.IPNet{net.IPv4zero, net.IPv4Mask(0, 0, 0, 0)}, - GW: net.ParseIP(hnsEndpoint.GatewayAddress), - }, - }, - } - - if nwCfg.IPV6Mode != "" && len(epInfo.IPAddresses) > 1 { - ipv6Config := &cniTypesCurr.IPConfig{ - Address: epInfo.IPAddresses[1], - } - - if len(nwInfo.Subnets) > 1 { - ipv6Config.Gateway = nwInfo.Subnets[1].Gateway - } - - result.IPs = append(result.IPs, ipv6Config) - } - - // Populate DNS servers. - result.DNS.Nameservers = nwCfg.DNS.Nameservers - return result, nil - } - - err = fmt.Errorf("GetHNSEndpointByName for %v returned nil with err %v", endpointId, err) - return nil, err -} - func addDefaultRoute(_ string, _ *network.EndpointInfo, _ *network.InterfaceInfo) { } diff --git a/network/endpoint.go b/network/endpoint.go index a5c36fa854..9b251b5e59 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -65,7 +65,7 @@ type EndpointInfo struct { EndpointID string ContainerID string NetNsPath string - IfName string + IfName string // value differs during creation vs. deletion flow SandboxKey string IfIndex int MacAddress net.HardwareAddr From 17c3fa03130ea7659a27bd3e84032337a2c110a3 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Sat, 25 May 2024 10:54:35 -0700 Subject: [PATCH 087/102] fix release ips when create a container without nictype using older cni and then upgrade cni and delete if we create a pod with an older cni version, it won't have a nictype if we upgrade cni and then delete, we should treat an empty nictype as an infra nictype and call the invoker delete --- cni/network/network.go | 2 +- cni/network/network_linux.go | 1 - network/manager.go | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 2e3ee2bcb7..4e3c906093 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -1127,7 +1127,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { zap.String("endpointID", epInfo.EndpointID)) sendEvent(plugin, fmt.Sprintf("Deleting endpoint:%v", epInfo.EndpointID)) - if !nwCfg.MultiTenancy && epInfo.NICType == cns.InfraNIC { + if !nwCfg.MultiTenancy && epInfo.NICType != cns.DelegatedVMNIC { // Delegated/secondary nic ips are statically allocated so we don't need to release // Call into IPAM plugin to release the endpoint's addresses. for i := range epInfo.IPAddresses { diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index ae1d0f234d..95c145e42b 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -8,7 +8,6 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/policy" - cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypesCurr "github.com/containernetworking/cni/pkg/types/100" ) diff --git a/network/manager.go b/network/manager.go index 0ac288a4ba..355b98cf86 100644 --- a/network/manager.go +++ b/network/manager.go @@ -792,6 +792,8 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI return ret } +// gets all endpoint infos associated with a container id and populates the network id field +// nictype may be empty in which case it is likely of type "infra" func (nm *networkManager) GetEndpointInfosFromContainerID(containerID string) []*EndpointInfo { ret := []*EndpointInfo{} for _, extIf := range nm.ExternalInterfaces { From 24a8396d486c5a943965929e1e2e17a7355d2665 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 28 May 2024 13:26:00 -0700 Subject: [PATCH 088/102] prevent eps with delegated nic type present on ep from also calling transparent endpoint client on delete tested on swift v2 linux single pod add, change cni to this version, delete (ok) then add using this cni version and delete, no extraneous transparent endpoint client calls logged --- network/endpoint_linux.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 04b4191e72..7290a3e531 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -292,6 +292,10 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. epClient.DeleteEndpointRules(ep) //nolint:errcheck // ignore error epClient.DeleteEndpoints(ep) + if ep.NICType == cns.DelegatedVMNIC { + // if the ep itself is of type secondary (new way), don't use transparent client below + return nil + } } epClient = NewTransparentEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, nioc, plc) From 8ed93a5fe5cbb735e537b92370ec87d329c6ed81 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 28 May 2024 17:36:18 -0700 Subject: [PATCH 089/102] mock get interface method for ut searched for "NetPlugin" in all files and determined all prod use of NetPlugin goes through NewNetPlugin where we set the get interface method to the real interface get method adds ut where the master interface (by mac) is not found --- cni/network/network.go | 11 ++++++-- cni/network/network_test.go | 51 ++++++++++++++++++++++++++++++++++++- netio/netio.go | 4 +++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 4e3c906093..d0ade208fd 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -76,6 +76,7 @@ type NetPlugin struct { tb *telemetry.TelemetryBuffer nnsClient NnsClient multitenancyClient MultitenancyClient + netClient InterfaceGetter } type PolicyArgs struct { @@ -97,6 +98,11 @@ type NnsClient interface { DeleteContainerNetworking(ctx context.Context, podName, nwNamespace string) (*nnscontracts.ConfigureContainerNetworkingResponse, error) } +// client for getting interface +type InterfaceGetter interface { + GetNetworkInterfaces() ([]net.Interface, error) +} + // snatConfiguration contains a bool that determines whether CNI enables snat on host and snat for dns type snatConfiguration struct { EnableSnatOnHost bool @@ -129,6 +135,7 @@ func NewPlugin(name string, nm: nm, nnsClient: client, multitenancyClient: multitenancyClient, + netClient: &netio.NetIO{}, }, nil } @@ -207,7 +214,7 @@ func (plugin *NetPlugin) Stop() { // findInterfaceByMAC returns the name of the master interface func (plugin *NetPlugin) findInterfaceByMAC(macAddress string) string { - interfaces, err := net.Interfaces() + interfaces, err := plugin.netClient.GetNetworkInterfaces() if err != nil { logger.Error("failed to get interfaces", zap.Error(err)) return "" @@ -235,7 +242,7 @@ func (plugin *NetPlugin) findMasterInterfaceBySubnet(nwCfg *cni.NetworkConfig, s // Otherwise, pick the first interface with an IP address in the given subnet. subnetPrefixString := subnetPrefix.String() - interfaces, err := net.Interfaces() + interfaces, err := plugin.netClient.GetNetworkInterfaces() if err != nil { logger.Error("failed to get interfaces", zap.Error(err)) return "" diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 8f92760e36..1f86a1fda5 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -1133,6 +1133,18 @@ func TestGetPodSubnetNatInfo(t *testing.T) { } } +type InterfaceGetterMock struct { + interfaces []net.Interface + err error +} + +func (n *InterfaceGetterMock) GetNetworkInterfaces() ([]net.Interface, error) { + if n.err != nil { + return nil, n.err + } + return n.interfaces, nil +} + func TestPluginSwiftV2Add(t *testing.T) { plugin, _ := cni.NewPlugin("name", "0.3.0") @@ -1159,6 +1171,11 @@ func TestPluginSwiftV2Add(t *testing.T) { ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, + netClient: &InterfaceGetterMock{ + interfaces: []net.Interface{ + {Name: "eth0"}, + }, + }, }, args: &cniSkel.CmdArgs{ StdinData: localNwCfg.Serialize(), @@ -1177,6 +1194,11 @@ func TestPluginSwiftV2Add(t *testing.T) { ipamInvoker: NewMockIpamInvoker(false, false, false, true, true), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, + netClient: &InterfaceGetterMock{ + interfaces: []net.Interface{ + {Name: "eth0"}, + }, + }, }, args: &cniSkel.CmdArgs{ StdinData: localNwCfg.Serialize(), @@ -1202,6 +1224,11 @@ func TestPluginSwiftV2Add(t *testing.T) { ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, + netClient: &InterfaceGetterMock{ + interfaces: []net.Interface{ + {Name: "eth0"}, + }, + }, }, args: &cniSkel.CmdArgs{ StdinData: localNwCfg.Serialize(), @@ -1213,6 +1240,28 @@ func TestPluginSwiftV2Add(t *testing.T) { wantErr: true, wantErrMsg: "failed to create endpoint: MockEndpointClient Error : AddEndpoints Delegated VM NIC failed", }, + { + name: "SwiftV2 Find Interface By MAC Address Fail", + plugin: &NetPlugin{ + Plugin: plugin, + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), + ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), + report: &telemetry.CNIReport{}, + tb: &telemetry.TelemetryBuffer{}, + netClient: &InterfaceGetterMock{ + interfaces: []net.Interface{}, + }, + }, + args: &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "test-container", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + wantErr: true, + wantErrMsg: "Failed to find the master interface", + }, } for _, tt := range tests { @@ -1221,7 +1270,7 @@ func TestPluginSwiftV2Add(t *testing.T) { err := tt.plugin.Add(tt.args) if tt.wantErr { require.Error(t, err) - assert.Equal(t, err.Error(), tt.wantErrMsg, "Expected %v but got %+v", tt.wantErrMsg, err.Error()) + assert.Equal(t, tt.wantErrMsg, err.Error(), "Expected %v but got %+v", tt.wantErrMsg, err.Error()) endpoints, _ := tt.plugin.nm.GetAllEndpoints(localNwCfg.Name) require.Condition(t, assert.Comparison(func() bool { return len(endpoints) == 0 })) } else { diff --git a/netio/netio.go b/netio/netio.go index 9462c8ed33..3976c33527 100644 --- a/netio/netio.go +++ b/netio/netio.go @@ -48,3 +48,7 @@ func (ns *NetIO) GetNetworkInterfaceByMac(mac net.HardwareAddr) (*net.Interface, return nil, ErrInterfaceNotFound } + +func (ns *NetIO) GetNetworkInterfaces() ([]net.Interface, error) { + return net.Interfaces() +} From b2bdd8c1b1b7f3fd235452af55b81f9e38012150 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 28 May 2024 17:50:57 -0700 Subject: [PATCH 090/102] address feedback (noop) --- network/endpoint_linux.go | 63 +++++++++++++++++++-------------------- network/manager.go | 9 +----- network/network.go | 1 - 3 files changed, 32 insertions(+), 41 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 7290a3e531..6854d2e347 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -60,33 +60,32 @@ func (nw *network) newEndpointImpl( epInfo *EndpointInfo, ) (*endpoint, error) { var ( - err error - hostIfName string - contIfName string - localIP string - vlanid = 0 - defaultEpInfo = epInfo - containerIf *net.Interface + err error + hostIfName string + contIfName string + localIP string + vlanid = 0 + containerIf *net.Interface ) - if nw.Endpoints[defaultEpInfo.EndpointID] != nil { + if nw.Endpoints[epInfo.EndpointID] != nil { logger.Info("[net] Endpoint already exists.") err = errEndpointExists return nil, err } - if defaultEpInfo.Data != nil { - if _, ok := defaultEpInfo.Data[VlanIDKey]; ok { - vlanid = defaultEpInfo.Data[VlanIDKey].(int) + if epInfo.Data != nil { + if _, ok := epInfo.Data[VlanIDKey]; ok { + vlanid = epInfo.Data[VlanIDKey].(int) } - if _, ok := defaultEpInfo.Data[LocalIPKey]; ok { - localIP = defaultEpInfo.Data[LocalIPKey].(string) + if _, ok := epInfo.Data[LocalIPKey]; ok { + localIP = epInfo.Data[LocalIPKey].(string) } } - if _, ok := defaultEpInfo.Data[OptVethName]; ok { - key := defaultEpInfo.Data[OptVethName].(string) + if _, ok := epInfo.Data[OptVethName]; ok { + key := epInfo.Data[OptVethName].(string) logger.Info("Generate veth name based on the key provided", zap.String("key", key)) vethname := generateVethName(key) hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) @@ -96,8 +95,8 @@ func (nw *network) newEndpointImpl( } else { // Create a veth pair. logger.Info("Generate veth name based on endpoint id") - hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, defaultEpInfo.EndpointID[:7]) - contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, defaultEpInfo.EndpointID[:7]) + hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, epInfo.EndpointID[:7]) + contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, epInfo.EndpointID[:7]) } nicName := epInfo.IfName @@ -107,27 +106,27 @@ func (nw *network) newEndpointImpl( } ep := &endpoint{ - Id: defaultEpInfo.EndpointID, + Id: epInfo.EndpointID, // IfName should end up being eth0 in non-delegated nic cases IfName: nicName, // container veth pair name. In cnm, we won't rename this and docker expects veth name. HostIfName: hostIfName, - InfraVnetIP: defaultEpInfo.InfraVnetIP, + InfraVnetIP: epInfo.InfraVnetIP, LocalIP: localIP, - IPAddresses: defaultEpInfo.IPAddresses, - DNS: defaultEpInfo.EndpointDNS, + IPAddresses: epInfo.IPAddresses, + DNS: epInfo.EndpointDNS, VlanID: vlanid, - EnableSnatOnHost: defaultEpInfo.EnableSnatOnHost, - EnableInfraVnet: defaultEpInfo.EnableInfraVnet, - EnableMultitenancy: defaultEpInfo.EnableMultiTenancy, - AllowInboundFromHostToNC: defaultEpInfo.AllowInboundFromHostToNC, - AllowInboundFromNCToHost: defaultEpInfo.AllowInboundFromNCToHost, - NetworkNameSpace: defaultEpInfo.NetNsPath, - ContainerID: defaultEpInfo.ContainerID, - PODName: defaultEpInfo.PODName, - PODNameSpace: defaultEpInfo.PODNameSpace, - Routes: defaultEpInfo.Routes, + EnableSnatOnHost: epInfo.EnableSnatOnHost, + EnableInfraVnet: epInfo.EnableInfraVnet, + EnableMultitenancy: epInfo.EnableMultiTenancy, + AllowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, + AllowInboundFromNCToHost: epInfo.AllowInboundFromNCToHost, + NetworkNameSpace: epInfo.NetNsPath, + ContainerID: epInfo.ContainerID, + PODName: epInfo.PODName, + PODNameSpace: epInfo.PODNameSpace, + Routes: epInfo.Routes, SecondaryInterfaces: make(map[string]*InterfaceInfo), - NICType: defaultEpInfo.NICType, + NICType: epInfo.NICType, } if nw.extIf != nil { ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} diff --git a/network/manager.go b/network/manager.go index 355b98cf86..95820897b4 100644 --- a/network/manager.go +++ b/network/manager.go @@ -725,14 +725,7 @@ func (nm *networkManager) SaveState(eps []*endpoint) error { // If we fail half way, we'll propagate an error up which should clean everything up if nm.IsStatelessCNIMode() { err := nm.UpdateEndpointState(eps) - if err != nil { - return err - } - } - - // we either use stateless cni and save via update endpoint state, or use the state file - if nm.IsStatelessCNIMode() { - return nil + return err } // once endpoints and networks are in-memory, save once diff --git a/network/network.go b/network/network.go index b82d56ad27..319f82b9a4 100644 --- a/network/network.go +++ b/network/network.go @@ -75,7 +75,6 @@ type NetworkInfo struct { IPAMType string ServiceCidrs string IsIPv6Enabled bool - NICType string } // SubnetInfo contains subnet information for a container network. From 07ce9d7e320ff3f2c4711d597210a541b8b5eb9f Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 29 May 2024 10:02:51 -0700 Subject: [PATCH 091/102] add ut for handling empty nictype on cns add (noop) --- cni/network/invoker_azure_test.go | 5 +- cni/network/invoker_cns_test.go | 79 +++++++++++++++++++++++++++++++ cni/network/multitenancy_test.go | 4 ++ 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 26b32fe514..a2bd8f5676 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -251,11 +251,14 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { require.Nil(err) } - for _, ifInfo := range ipamAddResult.interfaceInfo { + for key, ifInfo := range ipamAddResult.interfaceInfo { if ifInfo.NICType == cns.InfraNIC { fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ifInfo.IPConfigs) require.Exactly(tt.want, ifInfo.IPConfigs) } + // azure ipam invoker always sets key as infra nic + require.Equal(string(cns.InfraNIC), key) + require.Equal(cns.InfraNIC, ifInfo.NICType) } }) } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index b6f5d48730..307fee643d 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -599,6 +599,74 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { }, wantErr: false, }, + { + name: "Test CNI add with pod ip info empty nictype", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + DNSServers: nil, + GatewayIPAddress: "10.0.0.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + options: map[string]interface{}{}, + }, + wantDefaultResult: network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ + { + Address: *getCIDRNotationForAddress("10.0.1.10/24"), + Gateway: net.ParseIP("10.0.0.1"), + }, + }, + Routes: []network.RouteInfo{ + { + Dst: network.Ipv4DefaultRouteDstPrefix, + Gw: net.ParseIP("10.0.0.1"), + }, + }, + NICType: cns.InfraNIC, + HostSubnetPrefix: *parseCIDR("10.0.0.0/24"), + }, + wantErr: false, + }, { name: "Test happy CNI add for both ipv4 and ipv6", fields: fields{ @@ -732,6 +800,7 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { } for _, ifInfo := range ipamAddResult.interfaceInfo { + require.NotEqual("", string(ifInfo.NICType), "nictype should be auto populated if empty") if ifInfo.NICType == cns.DelegatedVMNIC { fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ifInfo) if len(tt.wantMultitenantResult.IPConfigs) > 0 { @@ -1367,3 +1436,13 @@ func Test_setHostOptions(t *testing.T) { }) } } + +func Test_getInterfaceInfoKey(t *testing.T) { + require := require.New(t) //nolint further usage of require without passing t + inv := &CNSIPAMInvoker{} + dummyMAC := "12:34:56:78:9a:bc" + require.Equal(string(cns.InfraNIC), inv.getInterfaceInfoKey(cns.InfraNIC, dummyMAC)) + require.Equal(dummyMAC, inv.getInterfaceInfoKey(cns.DelegatedVMNIC, dummyMAC)) + require.Equal("", inv.getInterfaceInfoKey(cns.DelegatedVMNIC, "")) + require.Equal(string(cns.BackendNIC), inv.getInterfaceInfoKey(cns.BackendNIC, dummyMAC)) +} diff --git a/cni/network/multitenancy_test.go b/cni/network/multitenancy_test.go index 42d2cb2540..4b730d4b2e 100644 --- a/cni/network/multitenancy_test.go +++ b/cni/network/multitenancy_test.go @@ -555,6 +555,9 @@ func TestGetMultiTenancyCNIResult(t *testing.T) { // check multiple responses tt.want5 = append(tt.want5, *tt.want1, *tt.want2) require.Exactly(tt.want5, ncResponses) + + require.Equal(cns.InfraNIC, got.interfaceInfo[string(cns.InfraNIC)+"0"].NICType) + require.Equal(cns.InfraNIC, got.interfaceInfo[string(cns.InfraNIC)+"1"].NICType) }) } } @@ -692,6 +695,7 @@ func TestGetMultiTenancyCNIResultUnsupportedAPI(t *testing.T) { } require.NoError(err) require.Exactly(tt.want, got.interfaceInfo[string(cns.InfraNIC)+"0"].NCResponse) + require.Equal(cns.InfraNIC, got.interfaceInfo[string(cns.InfraNIC)+"0"].NICType) }) } } From 209650cdad316e60bddde83563de19911c75c226 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 29 May 2024 13:16:04 -0700 Subject: [PATCH 092/102] add multitenancy delete net not found ut (noop) --- cni/network/network_test.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 1f86a1fda5..78ab9f446f 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -696,18 +696,41 @@ func TestPluginMultitenancyDelete(t *testing.T) { Master: "eth0", } + happyArgs := &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "test-container", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + } + tests := []struct { name string methods []string args *cniSkel.CmdArgs + delArgs *cniSkel.CmdArgs wantErr bool wantErrMsg string }{ { name: "Multitenancy delete success", methods: []string{CNI_ADD, CNI_DEL}, - args: &cniSkel.CmdArgs{ - StdinData: localNwCfg.Serialize(), + args: happyArgs, + delArgs: happyArgs, + wantErr: false, + }, + { + name: "Multitenancy delete net not found", + methods: []string{CNI_ADD, CNI_DEL}, + args: happyArgs, + delArgs: &cniSkel.CmdArgs{ + StdinData: (&cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "othernet", + MultiTenancy: true, + EnableExactMatchForPodName: true, + Master: "eth0", + }).Serialize(), ContainerID: "test-container", Netns: "test-container", Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), @@ -725,7 +748,7 @@ func TestPluginMultitenancyDelete(t *testing.T) { if method == CNI_ADD { err = plugin.Add(tt.args) } else if method == CNI_DEL { - err = plugin.Delete(tt.args) + err = plugin.Delete(tt.delArgs) } } if tt.wantErr { From 2f15ae45fe00a767deb942dcacb30cbdf5953d5d Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 29 May 2024 17:34:59 -0700 Subject: [PATCH 093/102] add uts for multi interface infos single add call, verify endpoint id, cns to cni data conversion and vice versa, get endpoint info from container id (noop) verifies partial success will delete all endpoints, even successfully created ones in the same cni add call --- cni/network/invoker_mock.go | 13 +++ cni/network/network_test.go | 139 ++++++++++++++++++++++ network/manager_mock.go | 21 +++- network/manager_test.go | 225 ++++++++++++++++++++++++++++++++++++ 4 files changed, 393 insertions(+), 5 deletions(-) diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index a89e0463a6..2731e1c66c 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -31,6 +31,7 @@ type MockIpamInvoker struct { delegatedVMNIC bool delegatedVMNICFail bool ipMap map[string]bool + customReturn map[string]network.InterfaceInfo } func NewMockIpamInvoker(ipv6, v4Fail, v6Fail, delegatedVMNIC, delegatedVMNICFail bool) *MockIpamInvoker { @@ -44,6 +45,13 @@ func NewMockIpamInvoker(ipv6, v4Fail, v6Fail, delegatedVMNIC, delegatedVMNICFail ipMap: make(map[string]bool), } } +func NewCustomMockIpamInvoker(customReturn map[string]network.InterfaceInfo) *MockIpamInvoker { + return &MockIpamInvoker{ + customReturn: customReturn, + + ipMap: make(map[string]bool), + } +} func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddResult, err error) { if invoker.v4Fail { @@ -104,6 +112,11 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes } } + if invoker.customReturn != nil { + ipamAddResult.interfaceInfo = invoker.customReturn + return ipamAddResult, nil + } + return ipamAddResult, nil } diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 78ab9f446f..a82e1e5eec 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -1304,3 +1304,142 @@ func TestPluginSwiftV2Add(t *testing.T) { }) } } + +func TestPluginSwiftV2MultipleAdd(t *testing.T) { + // checks cases where we create multiple endpoints in one call (also checks endpoint id) + // assumes we never get two infras created in one add call + plugin, _ := cni.NewPlugin("name", "0.3.0") + + localNwCfg := cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "swiftv2", + ExecutionMode: string(util.V4Overlay), + EnableExactMatchForPodName: true, + Master: "eth0", + } + + args := &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "test-container", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + } + + tests := []struct { + name string + plugin *NetPlugin + args *cniSkel.CmdArgs + wantErr bool + wantErrMsg string + wantNumEps int + validEpIDs map[string]struct{} + }{ + { + name: "SwiftV2 Add Infra and Delegated", + plugin: &NetPlugin{ + Plugin: plugin, + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), + ipamInvoker: NewCustomMockIpamInvoker(map[string]acnnetwork.InterfaceInfo{ + "eth0-1": { + NICType: cns.DelegatedVMNIC, + }, + "eth0": { + NICType: cns.InfraNIC, + }, + }), + report: &telemetry.CNIReport{}, + tb: &telemetry.TelemetryBuffer{}, + netClient: &InterfaceGetterMock{ + interfaces: []net.Interface{ + {Name: "eth0"}, + }, + }, + }, + args: args, + wantErr: false, + wantNumEps: 2, + }, + { + name: "SwiftV2 Add Two Delegated", + plugin: &NetPlugin{ + Plugin: plugin, + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), + ipamInvoker: NewCustomMockIpamInvoker(map[string]acnnetwork.InterfaceInfo{ + "eth0": { + NICType: cns.DelegatedVMNIC, + }, + "eth0-1": { + NICType: cns.DelegatedVMNIC, + }, + }), + report: &telemetry.CNIReport{}, + tb: &telemetry.TelemetryBuffer{}, + netClient: &InterfaceGetterMock{ + interfaces: []net.Interface{ + {Name: "eth0"}, + }, + }, + }, + args: args, + wantErr: false, + wantNumEps: 2, + }, + { + // creates 2 endpoints, the first succeeds, the second doesn't + // ensures that delete is called to clean up the first endpoint that succeeded + name: "SwiftV2 Partial Add fail", + plugin: &NetPlugin{ + Plugin: plugin, + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(func(ep *acnnetwork.EndpointInfo) error { + if ep.NICType == cns.DelegatedVMNIC { + return acnnetwork.NewErrorMockEndpointClient("AddEndpoints Delegated VM NIC failed") //nolint:wrapcheck // ignore wrapping for test + } + + return nil + })), + ipamInvoker: NewCustomMockIpamInvoker(map[string]acnnetwork.InterfaceInfo{ + "eth0": { + NICType: cns.InfraNIC, + }, + "eth0-1": { + NICType: cns.DelegatedVMNIC, + }, + }), + report: &telemetry.CNIReport{}, + tb: &telemetry.TelemetryBuffer{}, + netClient: &InterfaceGetterMock{ + interfaces: []net.Interface{ + {Name: "eth0"}, + }, + }, + }, + args: args, + wantNumEps: 0, + wantErr: true, + wantErrMsg: "failed to create endpoint: MockEndpointClient Error : AddEndpoints Delegated VM NIC failed", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.plugin.Add(tt.args) + if tt.wantErr { + require.Error(t, err) + assert.Equal(t, tt.wantErrMsg, err.Error(), "Expected %v but got %+v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + endpoints, _ := tt.plugin.nm.GetAllEndpoints(localNwCfg.Name) + require.Condition(t, assert.Comparison(func() bool { return len(endpoints) == tt.wantNumEps })) + for _, ep := range endpoints { + if ep.NICType == cns.InfraNIC { + require.Equal(t, "test-con-"+tt.args.IfName, ep.EndpointID, "infra nic must use ifname for its endpoint id") + } else { + require.Regexp(t, `test-con-\d+$`, ep.EndpointID, "other nics must use an index for their endpoint ids") + } + } + }) + } +} diff --git a/network/manager_mock.go b/network/manager_mock.go index c847a1a2c2..3217484a80 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -9,6 +9,7 @@ type MockNetworkManager struct { TestNetworkInfoMap map[string]*EndpointInfo TestEndpointInfoMap map[string]*EndpointInfo TestEndpointClient *MockEndpointClient + SaveStateMap map[string]*endpoint } // NewMockNetworkmanager returns a new mock @@ -17,6 +18,7 @@ func NewMockNetworkmanager(mockEndpointclient *MockEndpointClient) *MockNetworkM TestNetworkInfoMap: make(map[string]*EndpointInfo), TestEndpointInfoMap: make(map[string]*EndpointInfo), TestEndpointClient: mockEndpointclient, + SaveStateMap: make(map[string]*endpoint), } } @@ -154,8 +156,11 @@ func (nm *MockNetworkManager) GetNumEndpointsByContainerID(_ string) int { return numEndpoints } -func (nm *MockNetworkManager) SaveState(_ []*endpoint) error { - // TODO: Mock behavior for saving the state separate from TestEndpointInfo/NetworkInfo map maybe +func (nm *MockNetworkManager) SaveState(eps []*endpoint) error { + for _, ep := range eps { + nm.SaveStateMap[ep.Id] = ep + } + return nil } @@ -174,15 +179,21 @@ func (nm *MockNetworkManager) EndpointCreate(client apipaClient, epInfos []*Endp if err != nil { return err } - eps = append(eps, &endpoint{}) // mock append + eps = append(eps, &endpoint{ + Id: epInfo.EndpointID, + ContainerID: epInfo.ContainerID, + NICType: epInfo.NICType, + }) // mock append } // mock save endpoints return nm.SaveState(eps) } -func (nm *MockNetworkManager) DeleteState(_ []*EndpointInfo) error { - // TODO: Mock behavior for deleting the state separate from TestEndpointInfo/NetworkInfo map maybe +func (nm *MockNetworkManager) DeleteState(epInfos []*EndpointInfo) error { + for _, epInfo := range epInfos { + delete(nm.SaveStateMap, epInfo.EndpointID) + } return nil } diff --git a/network/manager_test.go b/network/manager_test.go index 567ef7c30c..b7ebf48ae9 100644 --- a/network/manager_test.go +++ b/network/manager_test.go @@ -2,12 +2,16 @@ package network import ( "errors" + "net" + "sort" "testing" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/store" "github.com/Azure/azure-container-networking/testutils" ) @@ -226,4 +230,225 @@ var _ = Describe("Test Manager", func() { }) }) }) + Describe("Test EndpointCreate", func() { + Context("When no endpoints provided", func() { + It("Should return 0", func() { + nm := &networkManager{} + err := nm.EndpointCreate(nil, []*EndpointInfo{}) + Expect(err).NotTo(HaveOccurred()) + num := nm.GetNumberOfEndpoints("", "") + Expect(num).To(Equal(0)) + }) + }) + }) + Describe("Test GetEndpointInfosFromContainerID", func() { + Context("When getting containers based on container id regardless of network", func() { + It("Should return 0", func() { + nm := &networkManager{ + ExternalInterfaces: map[string]*externalInterface{ + "eth0": { + Networks: map[string]*network{ + "azure": { + Endpoints: map[string]*endpoint{ + "12345678-eth0": { + Id: "12345678-eth0", + ContainerID: "12345678", + // potentially empty nictype + }, + "abcdefgh-eth0": { + Id: "abcdefgh-eth0", + ContainerID: "abcdefgh", + }, + }, + }, + "other": { + Endpoints: map[string]*endpoint{ + "12345678-1": { + Id: "12345678-1", + ContainerID: "12345678", + NICType: cns.DelegatedVMNIC, + }, + }, + }, + }, + }, + }, + } + epInfos := nm.GetEndpointInfosFromContainerID("12345678") + sort.Slice(epInfos, func(i, j int) bool { + return epInfos[i].EndpointID < epInfos[j].EndpointID + }) + Expect(len(epInfos)).To(Equal(2)) + + Expect(epInfos[0].EndpointID).To(Equal("12345678-1")) + Expect(epInfos[0].NICType).To(Equal(cns.DelegatedVMNIC)) + Expect(epInfos[0].NetworkID).To(Equal("other")) + + Expect(epInfos[1].EndpointID).To(Equal("12345678-eth0")) + Expect(string(epInfos[1].NICType)).To(Equal("")) + Expect(epInfos[1].NetworkID).To(Equal("azure")) + + }) + }) + }) + Describe("Test stateless cnsEndpointInfotoCNIEpInfos", func() { + endpointID := "" + _, dummyIP, _ := net.ParseCIDR("192.0.2.1/24") + dummyIPv4Slice := []net.IPNet{ + *dummyIP, + } + Context("When converting from cns to cni unmigrated", func() { + It("Should get the right cni endpoint info data", func() { + cnsEndpointInfo := restserver.EndpointInfo{ + IfnameToIPMap: map[string]*restserver.IPInfo{ + "": { + IPv4: dummyIPv4Slice, + HostVethName: "hostVeth1", + HnsEndpointID: "hnsID1", + HnsNetworkID: "hnsNetworkID1", + MacAddress: "12:34:56:78:9a:bc", + }, + }, + PodName: "test-pod", + PodNamespace: "test-pod-ns", + } + + epInfos := cnsEndpointInfotoCNIEpInfos(cnsEndpointInfo, endpointID) + + Expect(len(epInfos)).To(Equal(1)) + Expect(epInfos[0]).To(Equal( + &EndpointInfo{ + IPAddresses: dummyIPv4Slice, + IfName: InfraInterfaceName, + HostIfName: "hostVeth1", + HNSEndpointID: "hnsID1", + NICType: cns.InfraNIC, + HNSNetworkID: "hnsNetworkID1", + MacAddress: net.HardwareAddr("12:34:56:78:9a:bc"), + ContainerID: endpointID, + EndpointID: endpointID, + NetworkContainerID: endpointID, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + }, + ), "empty infos received from cns should be auto populated and treated as infra") + }) + }) + Context("When converting from cns to cni migrated", func() { + _, dummyIP2, _ := net.ParseCIDR("193.0.2.1/24") + dummyIPv4Slice2 := []net.IPNet{ + *dummyIP2, + } + It("Should get the right cni endpoint info data if there are multiple ip infos", func() { + cnsEndpointInfo := restserver.EndpointInfo{ + IfnameToIPMap: map[string]*restserver.IPInfo{ + "ifName1": { + IPv4: dummyIPv4Slice, + HostVethName: "hostVeth1", + HnsEndpointID: "hnsID1", + HnsNetworkID: "hnsNetworkID1", + MacAddress: "12:34:56:78:9a:bc", + NICType: cns.InfraNIC, + }, + "ifName2": { + IPv4: dummyIPv4Slice2, + HostVethName: "hostVeth2", + HnsEndpointID: "hnsID2", + HnsNetworkID: "hnsNetworkID2", + MacAddress: "22:34:56:78:9a:bc", + NICType: cns.DelegatedVMNIC, + }, + }, + PodName: "test-pod", + PodNamespace: "test-pod-ns", + } + + epInfos := cnsEndpointInfotoCNIEpInfos(cnsEndpointInfo, endpointID) + + Expect(len(epInfos)).To(Equal(2)) + Expect(epInfos[0]).To(Equal( + &EndpointInfo{ + IPAddresses: dummyIPv4Slice, + IfName: "ifName1", + HostIfName: "hostVeth1", + HNSEndpointID: "hnsID1", + NICType: cns.InfraNIC, + HNSNetworkID: "hnsNetworkID1", + MacAddress: net.HardwareAddr("12:34:56:78:9a:bc"), + ContainerID: endpointID, + EndpointID: endpointID, + NetworkContainerID: endpointID, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + }, + )) + Expect(epInfos[1]).To(Equal( + &EndpointInfo{ + IPAddresses: dummyIPv4Slice2, + IfName: "ifName2", + HostIfName: "hostVeth2", + HNSEndpointID: "hnsID2", + NICType: cns.DelegatedVMNIC, + HNSNetworkID: "hnsNetworkID2", + MacAddress: net.HardwareAddr("22:34:56:78:9a:bc"), + ContainerID: endpointID, + EndpointID: endpointID, + NetworkContainerID: endpointID, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + }, + )) + }) + }) + }) + Describe("Test stateless generateCNSIPInfoMap", func() { + Context("When converting from cni to cns", func() { + It("Should generate the cns endpoint info data from the endpoint structs", func() { + mac1, _ := net.ParseMAC("12:34:56:78:9a:bc") + mac2, _ := net.ParseMAC("22:34:56:78:9a:bc") + endpoints := []*endpoint{ + { + IfName: "eth0", + NICType: cns.InfraNIC, + HnsId: "hnsEndpointID1", + HNSNetworkID: "hnsNetworkID1", + HostIfName: "hostIfName1", + MacAddress: mac1, + }, + { + IfName: "eth1", + NICType: cns.DelegatedVMNIC, + HnsId: "hnsEndpointID2", + HNSNetworkID: "hnsNetworkID2", + HostIfName: "hostIfName2", + MacAddress: mac2, + }, + } + cnsEpInfos := generateCNSIPInfoMap(endpoints) + Expect(len(cnsEpInfos)).To(Equal(2)) + + Expect(cnsEpInfos).To(HaveKey("eth0")) + Expect(cnsEpInfos["eth0"]).To(Equal( + &restserver.IPInfo{ + NICType: cns.InfraNIC, + HnsEndpointID: "hnsEndpointID1", + HnsNetworkID: "hnsNetworkID1", + HostVethName: "hostIfName1", + MacAddress: "12:34:56:78:9a:bc", + }, + )) + + Expect(cnsEpInfos).To(HaveKey("eth1")) + Expect(cnsEpInfos["eth1"]).To(Equal( + &restserver.IPInfo{ + NICType: cns.DelegatedVMNIC, + HnsEndpointID: "hnsEndpointID2", + HnsNetworkID: "hnsNetworkID2", + HostVethName: "hostIfName2", + MacAddress: "22:34:56:78:9a:bc", + }, + )) + }) + }) + }) }) From fd45d0cee692d6d92ccde82a0a73a46c916ace34 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 29 May 2024 18:43:11 -0700 Subject: [PATCH 094/102] add ut for all pods associated with container id delete in one del call, new secondary delete flow (noop) --- cni/network/network_test.go | 7 ++++- netio/netio.go | 3 +- network/manager_test.go | 1 - network/secondary_endpoint_linux_test.go | 35 ++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/cni/network/network_test.go b/cni/network/network_test.go index a82e1e5eec..ca3a9c8ea4 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -1305,7 +1305,7 @@ func TestPluginSwiftV2Add(t *testing.T) { } } -func TestPluginSwiftV2MultipleAdd(t *testing.T) { +func TestPluginSwiftV2MultipleAddDelete(t *testing.T) { // checks cases where we create multiple endpoints in one call (also checks endpoint id) // assumes we never get two infras created in one add call plugin, _ := cni.NewPlugin("name", "0.3.0") @@ -1440,6 +1440,11 @@ func TestPluginSwiftV2MultipleAdd(t *testing.T) { require.Regexp(t, `test-con-\d+$`, ep.EndpointID, "other nics must use an index for their endpoint ids") } } + + err = tt.plugin.Delete(tt.args) + require.NoError(t, err) + endpoints, _ = tt.plugin.nm.GetAllEndpoints(localNwCfg.Name) + require.Condition(t, assert.Comparison(func() bool { return len(endpoints) == 0 })) }) } } diff --git a/netio/netio.go b/netio/netio.go index 3976c33527..13937b9577 100644 --- a/netio/netio.go +++ b/netio/netio.go @@ -50,5 +50,6 @@ func (ns *NetIO) GetNetworkInterfaceByMac(mac net.HardwareAddr) (*net.Interface, } func (ns *NetIO) GetNetworkInterfaces() ([]net.Interface, error) { - return net.Interfaces() + ifs, err := net.Interfaces() + return ifs, errors.Wrap(err, "GetNetworkInterfaces failed") } diff --git a/network/manager_test.go b/network/manager_test.go index b7ebf48ae9..8cc750cc66 100644 --- a/network/manager_test.go +++ b/network/manager_test.go @@ -287,7 +287,6 @@ var _ = Describe("Test Manager", func() { Expect(epInfos[1].EndpointID).To(Equal("12345678-eth0")) Expect(string(epInfos[1].NICType)).To(Equal("")) Expect(epInfos[1].NetworkID).To(Equal("azure")) - }) }) }) diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index 775ffeeb6a..6073101b32 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -7,6 +7,7 @@ import ( "net" "testing" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" @@ -185,6 +186,40 @@ func TestSecondaryDeleteEndpoints(t *testing.T) { }, wantErr: true, }, + { + // new way to handle delegated nics + // if the nictype is delegated, the data is on the endpoint itself, not the secondary interfaces field + name: "Delete endpoint with nic type delegated", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + // revisit in future, but currently the struct looks like this (with duplicated fields) + ep: &endpoint{ + NetworkNameSpace: "testns", + IfName: "eth1", + Routes: []RouteInfo{ + { + Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, + }, + }, + NICType: cns.DelegatedVMNIC, + SecondaryInterfaces: map[string]*InterfaceInfo{ + "eth1": { + Name: "eth1", + Routes: []RouteInfo{ + { + Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, + }, + }, + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { From bde0294450eb162bf5552f3077bcf1e8b80da0cc Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 30 May 2024 10:03:47 -0400 Subject: [PATCH 095/102] add two UTs --- cni/network/invoker_cns_test.go | 112 ++++++++++++++++++++++++++++ cni/network/network_windows_test.go | 62 +++++++++++++++ 2 files changed, 174 insertions(+) diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 307fee643d..2361c0b0d6 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -1446,3 +1446,115 @@ func Test_getInterfaceInfoKey(t *testing.T) { require.Equal("", inv.getInterfaceInfoKey(cns.DelegatedVMNIC, "")) require.Equal(string(cns.BackendNIC), inv.getInterfaceInfoKey(cns.BackendNIC, dummyMAC)) } + +func TestCNSIPAMInvoker_Add_SwiftV2(t *testing.T) { + require := require.New(t) //nolint further usage of require without passing t + + macAddress := "12:34:56:78:9a:bc" + parsedMacAddress, _ := net.ParseMAC(macAddress) + + type fields struct { + podName string + podNamespace string + cnsClient cnsclient + } + + type args struct { + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + hostSubnetPrefix *net.IPNet + options map[string]interface{} + } + + tests := []struct { + name string + fields fields + args args + wantSecondaryInterfacesInfo map[string]network.InterfaceInfo + wantErr bool + }{ + { + name: "Test happy CNI add with swiftv2 multitenant result", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname1", + InfraContainerID: "testcontainerid1", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.1.1.10", + PrefixLength: 24, + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.2", + Subnet: "10.0.0.1/24", + }, + NICType: cns.DelegatedVMNIC, + MacAddress: macAddress, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid1", + Netns: "testnetns1", + IfName: "testifname1", + }, + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + options: map[string]interface{}{}, + }, + wantSecondaryInterfacesInfo: map[string]network.InterfaceInfo{ + macAddress: network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ + { + Address: *getCIDRNotationForAddress("10.1.1.10/24"), + }, + }, + Routes: []network.RouteInfo{}, + NICType: cns.DelegatedVMNIC, + MacAddress: parsedMacAddress, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + invoker := &CNSIPAMInvoker{ + podName: tt.fields.podName, + podNamespace: tt.fields.podNamespace, + cnsClient: tt.fields.cnsClient, + } + ipamAddResult, err := invoker.Add(IPAMAddConfig{nwCfg: tt.args.nwCfg, args: tt.args.args, options: tt.args.options}) + if tt.wantErr { + require.Error(err) + } else { + require.NoError(err) + } + + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ipamAddResult.interfaceInfo) + if len(tt.wantSecondaryInterfacesInfo[macAddress].IPConfigs) > 0 { + require.EqualValues(tt.wantSecondaryInterfacesInfo, ipamAddResult.interfaceInfo, "incorrect multitenant response") + } + }) + } +} diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index e37a289344..b639b20417 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -521,3 +521,65 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }) } } + +func TestGetNetworkNameSwiftv2FromCNS(t *testing.T) { + // TODO: Add IB and Accelnet NIC test to this test + plugin, _ := cni.NewPlugin("name", "0.3.0") + + macAddress := "00:00:5e:00:53:01" + parsedMacAddress, _ := net.ParseMAC(macAddress) + swiftv2L1VHSecondaryInterfacesInfo := make(map[string]network.InterfaceInfo) + + swiftv2L1VHInterfaceInfo := network.InterfaceInfo{ + Name: "swiftv2L1VHinterface", + MacAddress: parsedMacAddress, + NICType: cns.DelegatedVMNIC, + } + swiftv2L1VHSecondaryInterfacesInfo[macAddress] = swiftv2L1VHInterfaceInfo + + tests := []struct { + name string + plugin *NetPlugin + netNs string + nwCfg *cni.NetworkConfig + ipamAddResult *IPAMAddResult + want net.HardwareAddr + wantErr bool + }{ + { + name: "Get Network Name from CNS for swiftv2", + plugin: &NetPlugin{ + Plugin: plugin, + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), + ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), + report: &telemetry.CNIReport{}, + tb: &telemetry.TelemetryBuffer{}, + }, + netNs: "azure", + nwCfg: &cni.NetworkConfig{ + CNIVersion: "0.3.0", + MultiTenancy: false, + }, + ipamAddResult: &IPAMAddResult{ + interfaceInfo: swiftv2L1VHSecondaryInterfacesInfo, + }, + want: parsedMacAddress, + wantErr: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Log(tt.ipamAddResult.interfaceInfo) + networkName, err := tt.plugin.getNetworkName(tt.netNs, &swiftv2L1VHInterfaceInfo, tt.nwCfg) + if tt.wantErr { + require.Error(t, err) + } else { + expectedMacAddress := "azure-" + tt.want.String() + require.NoError(t, err) + require.Equal(t, expectedMacAddress, networkName) + } + }) + } +} From c877fd4ecf52a3dba94cf9f71478578f1e70a550 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 30 May 2024 10:13:39 -0400 Subject: [PATCH 096/102] fix a linter issue --- cni/network/invoker_cns_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 2361c0b0d6..a12032ba2f 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -1522,7 +1522,7 @@ func TestCNSIPAMInvoker_Add_SwiftV2(t *testing.T) { options: map[string]interface{}{}, }, wantSecondaryInterfacesInfo: map[string]network.InterfaceInfo{ - macAddress: network.InterfaceInfo{ + macAddress: { IPConfigs: []*network.IPConfig{ { Address: *getCIDRNotationForAddress("10.1.1.10/24"), From 9bd874593776fa0b7c2b674cc34120519b649f24 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 30 May 2024 10:25:21 -0700 Subject: [PATCH 097/102] add ut to check endpoint ifname on new endpoint creation based on nictype (noop) --- network/endpoint_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/network/endpoint_test.go b/network/endpoint_test.go index 724094da27..0da1a8cb2d 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -178,6 +178,7 @@ var _ = Describe("Test Endpoint", func() { EndpointID: "768e8deb-eth1", Data: make(map[string]interface{}), IfName: eth0IfName, + NICType: cns.InfraNIC, } epInfo.Data[VlanIDKey] = 100 @@ -204,6 +205,7 @@ var _ = Describe("Test Endpoint", func() { Expect(len(ep.Gateways)).To(Equal(1)) Expect(ep.Gateways[0].String()).To(Equal("192.168.0.1")) Expect(ep.VlanID).To(Equal(epInfo.Data[VlanIDKey].(int))) + Expect(ep.IfName).To(Equal(epInfo.IfName)) }) It("Should be not added", func() { // Adding an endpoint with an id. @@ -236,6 +238,33 @@ var _ = Describe("Test Endpoint", func() { Expect(len(mockCli.endpoints)).To(Equal(0)) }) }) + Context("When endpoint added delegated", func() { + epInfo := &EndpointInfo{ + EndpointID: "768e8deb-eth1", + Data: make(map[string]interface{}), + IfName: eth0IfName, + MasterIfName: "masterIfName", + NICType: cns.DelegatedVMNIC, + } + epInfo.Data[VlanIDKey] = 100 + + It("should have fields set", func() { + nw2 := &network{ + Endpoints: map[string]*endpoint{}, + extIf: &externalInterface{IPv4Gateway: net.ParseIP("192.168.0.1")}, + } + ep, err := nw2.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), epInfo) + Expect(err).NotTo(HaveOccurred()) + Expect(ep).NotTo(BeNil()) + Expect(ep.Id).To(Equal(epInfo.EndpointID)) + Expect(ep.Gateways).NotTo(BeNil()) + Expect(len(ep.Gateways)).To(Equal(1)) + Expect(ep.Gateways[0].String()).To(Equal("192.168.0.1")) + Expect(ep.VlanID).To(Equal(epInfo.Data[VlanIDKey].(int))) + Expect(ep.IfName).To(Equal("masterIfName")) + }) + }) Context("When endpoint add failed", func() { It("Should not be added to the network", func() { nw := &network{ From 8f9f9cb96535609bdf775640a78fd4bb3f60d196 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 30 May 2024 14:45:46 -0700 Subject: [PATCH 098/102] add ut for fail to find interface by subnet (noop) --- cni/network/network_test.go | 55 ++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/cni/network/network_test.go b/cni/network/network_test.go index ca3a9c8ea4..68a22d7bf3 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -1179,6 +1179,14 @@ func TestPluginSwiftV2Add(t *testing.T) { Master: "eth0", } + args := &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "test-container", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + } + tests := []struct { name string plugin *NetPlugin @@ -1200,13 +1208,7 @@ func TestPluginSwiftV2Add(t *testing.T) { }, }, }, - args: &cniSkel.CmdArgs{ - StdinData: localNwCfg.Serialize(), - ContainerID: "test-container", - Netns: "test-container", - Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), - IfName: eth0IfName, - }, + args: args, wantErr: false, }, { @@ -1223,13 +1225,7 @@ func TestPluginSwiftV2Add(t *testing.T) { }, }, }, - args: &cniSkel.CmdArgs{ - StdinData: localNwCfg.Serialize(), - ContainerID: "test-container", - Netns: "test-container", - Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), - IfName: eth0IfName, - }, + args: args, wantErr: true, wantErrMsg: "IPAM Invoker Add failed with error: failed to add ipam invoker: delegatedVMNIC fail", }, @@ -1253,13 +1249,7 @@ func TestPluginSwiftV2Add(t *testing.T) { }, }, }, - args: &cniSkel.CmdArgs{ - StdinData: localNwCfg.Serialize(), - ContainerID: "test-container", - Netns: "test-container", - Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), - IfName: eth0IfName, - }, + args: args, wantErr: true, wantErrMsg: "failed to create endpoint: MockEndpointClient Error : AddEndpoints Delegated VM NIC failed", }, @@ -1275,8 +1265,29 @@ func TestPluginSwiftV2Add(t *testing.T) { interfaces: []net.Interface{}, }, }, + args: args, + wantErr: true, + wantErrMsg: "Failed to find the master interface", + }, + { + name: "SwiftV2 Find Interface By Subnet Prefix Fail", + plugin: &NetPlugin{ + Plugin: plugin, + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), + report: &telemetry.CNIReport{}, + tb: &telemetry.TelemetryBuffer{}, + netClient: &InterfaceGetterMock{ + interfaces: []net.Interface{}, + }, + }, args: &cniSkel.CmdArgs{ - StdinData: localNwCfg.Serialize(), + StdinData: (&cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "swiftv2", + ExecutionMode: string(util.V4Overlay), + EnableExactMatchForPodName: true, + }).Serialize(), ContainerID: "test-container", Netns: "test-container", Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), From cafa8d2099cc64ac97d8afbb267e7a201b3c2554 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 30 May 2024 15:40:30 -0700 Subject: [PATCH 099/102] Adding support for Stateless CNI Delete Edge case when there in no HNS ID --- cni/network/network.go | 12 +++++------- network/manager.go | 16 ++++++++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index d0ade208fd..7f47f7a1e4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -1099,13 +1099,11 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { logger.Error("Failed to query endpoint", zap.String("endpoint", endpointID), zap.Error(err)) - if !plugin.nm.IsStatelessCNIMode() { - logger.Error("Release ip by ContainerID (endpoint not found)", - zap.String("containerID", args.ContainerID)) - sendEvent(plugin, fmt.Sprintf("Release ip by ContainerID (endpoint not found):%v", args.ContainerID)) - if err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options); err != nil { - return plugin.RetriableError(fmt.Errorf("failed to release address(no endpoint): %w", err)) - } + logger.Error("Release ip by ContainerID (endpoint not found)", + zap.String("containerID", args.ContainerID)) + sendEvent(plugin, fmt.Sprintf("Release ip by ContainerID (endpoint not found):%v", args.ContainerID)) + if err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options); err != nil { + return plugin.RetriableError(fmt.Errorf("failed to release address(no endpoint): %w", err)) } } // Log the error but return success if the endpoint being deleted is not found. diff --git a/network/manager.go b/network/manager.go index 95820897b4..6f855cdc97 100644 --- a/network/manager.go +++ b/network/manager.go @@ -447,7 +447,7 @@ func (nm *networkManager) GetEndpointState(networkID, containerID string) ([]*En } epInfos[i], err = epInfos[i].GetEndpointInfoByIPImpl(epInfos[i].IPAddresses, networkID) if err != nil { - return nil, errors.Wrapf(err, "Get endpoint API returned with error") + logger.Info("Endpoint State is incomlete for endpoint: ", zap.Error(err), zap.String("endpointID", epInfos[i].EndpointID)) } } } @@ -511,19 +511,23 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint IfName: epInfo.IfName, // TODO: For stateless cni linux populate IfName here to use in deletion in secondary endpoint client } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) - err := nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) - if err != nil { - return err + // do not need to Delete HNS endpoint if the there is no HNS in state + if ep.HnsId != "" { + err := nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) + if err != nil { + return err + } } if epInfo.NICType == cns.DelegatedVMNIC { // we are currently assuming stateless is not running in linux // CHECK: could this affect linux? (if it does, it could disconnect external interface, is that okay?) // bad only when 1) stateless and 2) linux and 3) delegated vmnics exist logger.Info("Deleting endpoint because delegated vmnic detected", zap.String("HNSNetworkID", nw.HnsId)) - err = nm.deleteNetworkImpl(nw) + err := nm.deleteNetworkImpl(nw) // no need to clean up state in stateless + return err } - return err + return nil } // GetEndpointInfo returns information about the given endpoint. From 78162bed2831365e4f56072730b9f91c3a2d7eab Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 30 May 2024 22:33:08 -0400 Subject: [PATCH 100/102] fix uts --- cni/network/invoker_azure_test.go | 8 ++++---- cni/network/invoker_mock.go | 3 +-- cni/network/network_windows_test.go | 9 +++++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index a2bd8f5676..e41ee7171e 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -125,18 +125,18 @@ func getResult(ips ...string) []*network.IPConfig { } func getNwInfo(subnetv4, subnetv6 string) *network.EndpointInfo { - nwinfo := &network.EndpointInfo{} + nwInfo := &network.EndpointInfo{} if subnetv4 != "" { - nwinfo.Subnets = append(nwinfo.Subnets, network.SubnetInfo{ + nwInfo.Subnets = append(nwInfo.Subnets, network.SubnetInfo{ Prefix: *getCIDRNotationForAddress(subnetv4), }) } if subnetv6 != "" { - nwinfo.Subnets = append(nwinfo.Subnets, network.SubnetInfo{ + nwInfo.Subnets = append(nwInfo.Subnets, network.SubnetInfo{ Prefix: *getCIDRNotationForAddress(subnetv6), }) } - return nwinfo + return nwInfo } func TestAzureIPAMInvoker_Add(t *testing.T) { diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 2731e1c66c..bba3cd7c36 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -41,8 +41,7 @@ func NewMockIpamInvoker(ipv6, v4Fail, v6Fail, delegatedVMNIC, delegatedVMNICFail v6Fail: v6Fail, delegatedVMNIC: delegatedVMNIC, delegatedVMNICFail: delegatedVMNICFail, - - ipMap: make(map[string]bool), + ipMap: make(map[string]bool), } } func NewCustomMockIpamInvoker(customReturn map[string]network.InterfaceInfo) *MockIpamInvoker { diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index b639b20417..b8697334c3 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -580,6 +580,15 @@ func TestGetNetworkNameSwiftv2FromCNS(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedMacAddress, networkName) } + + networkID, err := tt.plugin.getNetworkID(tt.netNs, &swiftv2L1VHInterfaceInfo, tt.nwCfg) + if tt.wantErr { + require.Error(t, err) + } else { + expectedMacAddress := "azure-" + tt.want.String() + require.NoError(t, err) + require.Equal(t, expectedMacAddress, networkID) + } }) } } From ff234e9e12d9aedd70d25a3c9bb27ba186633ebb Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 30 May 2024 22:47:24 -0400 Subject: [PATCH 101/102] fix linter issues --- cni/network/invoker_mock.go | 1 + cni/network/network_windows.go | 3 ++- cni/network/network_windows_test.go | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index bba3cd7c36..1461436260 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -44,6 +44,7 @@ func NewMockIpamInvoker(ipv6, v4Fail, v6Fail, delegatedVMNIC, delegatedVMNICFail ipMap: make(map[string]bool), } } + func NewCustomMockIpamInvoker(customReturn map[string]network.InterfaceInfo) *MockIpamInvoker { return &MockIpamInvoker{ customReturn: customReturn, diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 417bbd8532..6ebfecb12d 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -72,9 +72,10 @@ func (plugin *NetPlugin) getNetworkName(netNs string, interfaceInfo *network.Int determineWinVer() // Swiftv2 L1VH Network Name + swiftv2NetworkNamePrefix := "azure-" if interfaceInfo != nil && interfaceInfo.NICType == cns.DelegatedVMNIC { logger.Info("swiftv2", zap.String("network name", interfaceInfo.MacAddress.String())) - return "azure-" + interfaceInfo.MacAddress.String(), nil + return swiftv2NetworkNamePrefix + interfaceInfo.MacAddress.String(), nil } // For singletenancy, the network name is simply the nwCfg.Name diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index b8697334c3..cb0240559f 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -527,6 +527,7 @@ func TestGetNetworkNameSwiftv2FromCNS(t *testing.T) { plugin, _ := cni.NewPlugin("name", "0.3.0") macAddress := "00:00:5e:00:53:01" + swiftv2NetworkNamePrefix := "azure-" parsedMacAddress, _ := net.ParseMAC(macAddress) swiftv2L1VHSecondaryInterfacesInfo := make(map[string]network.InterfaceInfo) @@ -576,7 +577,7 @@ func TestGetNetworkNameSwiftv2FromCNS(t *testing.T) { if tt.wantErr { require.Error(t, err) } else { - expectedMacAddress := "azure-" + tt.want.String() + expectedMacAddress := swiftv2NetworkNamePrefix + tt.want.String() require.NoError(t, err) require.Equal(t, expectedMacAddress, networkName) } @@ -585,7 +586,7 @@ func TestGetNetworkNameSwiftv2FromCNS(t *testing.T) { if tt.wantErr { require.Error(t, err) } else { - expectedMacAddress := "azure-" + tt.want.String() + expectedMacAddress := swiftv2NetworkNamePrefix + tt.want.String() require.NoError(t, err) require.Equal(t, expectedMacAddress, networkID) } From 87aefd048771fdb9059ef12d9d423cea880cbf8d Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 31 May 2024 13:36:08 -0700 Subject: [PATCH 102/102] fix ut --- network/manager_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/manager_test.go b/network/manager_test.go index 8cc750cc66..09569c25ba 100644 --- a/network/manager_test.go +++ b/network/manager_test.go @@ -365,7 +365,7 @@ var _ = Describe("Test Manager", func() { epInfos := cnsEndpointInfotoCNIEpInfos(cnsEndpointInfo, endpointID) Expect(len(epInfos)).To(Equal(2)) - Expect(epInfos[0]).To(Equal( + Expect(epInfos).To(ContainElement( &EndpointInfo{ IPAddresses: dummyIPv4Slice, IfName: "ifName1", @@ -381,7 +381,7 @@ var _ = Describe("Test Manager", func() { PODNameSpace: "test-pod-ns", }, )) - Expect(epInfos[1]).To(Equal( + Expect(epInfos).To(ContainElement( &EndpointInfo{ IPAddresses: dummyIPv4Slice2, IfName: "ifName2",