chore: Backport VNet Scale Changes into 1.4 release of CNS

cns/ipampool/monitor.go

@@ -3,6 +3,7 @@ package ipampool
 import (
 	"context"
 	"fmt"
+	"net/netip"
 	"strconv"
 	"sync"
 	"time"
@@ -124,9 +125,17 @@ func (pm *Monitor) Start(ctx context.Context) error {
 			// Add Primary IP to Map, if not present.
 			// This is only for Swift i.e. if NC Type is vnet.
 			for i := 0; i < len(nnc.Status.NetworkContainers); i++ {
-				if nnc.Status.NetworkContainers[i].Type == "" ||
-					nnc.Status.NetworkContainers[i].Type == v1alpha.VNET {
-					pm.metastate.primaryIPAddresses[nnc.Status.NetworkContainers[i].PrimaryIP] = struct{}{}
+				nc := nnc.Status.NetworkContainers[i]
+				if nc.Type == "" || nc.Type == v1alpha.VNET {
+					pm.metastate.primaryIPAddresses[nc.PrimaryIP] = struct{}{}
+				}
+
+				if nc.Type == v1alpha.VNETBlock {
+					primaryPrefix, err := netip.ParsePrefix(nc.PrimaryIP)
+					if err != nil {
+						return errors.Wrapf(err, "unable to parse ip prefix: %s", nc.PrimaryIP)
+					}
+					pm.metastate.primaryIPAddresses[primaryPrefix.Addr().String()] = struct{}{}
 				}
 			}
 

cns/kubecontroller/nodenetworkconfig/conversion.go

@@ -74,7 +74,10 @@ func CreateNCRequestFromDynamicNC(nc v1alpha.NetworkContainer) (*cns.CreateNetwo
 //
 //nolint:gocritic //ignore hugeparam
 func CreateNCRequestFromStaticNC(nc v1alpha.NetworkContainer) (*cns.CreateNetworkContainerRequest, error) {
-	nc.Version = 0 // fix for NMA always giving us version 0 for Overlay NCs
+	if nc.Type == v1alpha.Overlay {
+		nc.Version = 0 // fix for NMA always giving us version 0 for Overlay NCs
+	}
+
 	primaryPrefix, err := netip.ParsePrefix(nc.PrimaryIP)
 	if err != nil {
 		return nil, errors.Wrapf(err, "IP: %s", nc.PrimaryIP)
@@ -89,6 +92,9 @@ func CreateNCRequestFromStaticNC(nc v1alpha.NetworkContainer) (*cns.CreateNetwor
 		PrefixLength: uint8(subnetPrefix.Bits()),
 	}
 
-	req := createNCRequestFromStaticNCHelper(nc, primaryPrefix, subnet)
-	return req, nil
+	req, err := createNCRequestFromStaticNCHelper(nc, primaryPrefix, subnet)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error while creating NC request from static NC")
+	}
+	return req, err
 }

cns/kubecontroller/nodenetworkconfig/conversion_linux.go

@@ -6,13 +6,14 @@ import (
 
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
+	"github.com/pkg/errors"
 )
 
 // createNCRequestFromStaticNCHelper generates a CreateNetworkContainerRequest from a static NetworkContainer
 // by adding all IPs in the the block to the secondary IP configs list. It does not skip any IPs.
 //
 //nolint:gocritic //ignore hugeparam
-func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) *cns.CreateNetworkContainerRequest {
+func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) (*cns.CreateNetworkContainerRequest, error) {
 	secondaryIPConfigs := map[string]cns.SecondaryIPConfig{}
 
 	// iterate through all IP addresses in the subnet described by primaryPrefix and
@@ -23,6 +24,29 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			NCVersion: int(nc.Version),
 		}
 	}
+
+	// Add IPs from CIDR block to the secondary IPConfigs
+	if nc.Type == v1alpha.VNETBlock {
+		// Delete primary IP reserved for Primary IP for NC
+		delete(secondaryIPConfigs, primaryIPPrefix.Addr().String())
+
+		for _, ipAssignment := range nc.IPAssignments {
+			cidrPrefix, err := netip.ParsePrefix(ipAssignment.IP)
+			if err != nil {
+				return nil, errors.Wrapf(err, "invalid CIDR block: %s", ipAssignment.IP)
+			}
+
+			// iterate through all IP addresses in the CIDR block described by cidrPrefix and
+			// add them to the request as secondary IPConfigs.
+			for addr := cidrPrefix.Masked().Addr(); cidrPrefix.Contains(addr); addr = addr.Next() {
+				secondaryIPConfigs[addr.String()] = cns.SecondaryIPConfig{
+					IPAddress: addr.String(),
+					NCVersion: int(nc.Version),
+				}
+			}
+		}
+	}
+
 	return &cns.CreateNetworkContainerRequest{
 		SecondaryIPConfigs:   secondaryIPConfigs,
 		NetworkContainerid:   nc.ID,
@@ -32,6 +56,5 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
-		NCStatus: nc.Status,
-	}
+	}, nil
 }

cns/kubecontroller/nodenetworkconfig/conversion_test.go

@@ -10,19 +10,27 @@ import (
 )
 
 const (
-	uuid               = "539970a2-c2dd-11ea-b3de-0242ac130004"
-	defaultGateway     = "10.0.0.2"
-	ipIsCIDR           = "10.0.0.1/32"
-	ipMalformed        = "10.0.0.0.0"
-	ncID               = "160005ba-cd02-11ea-87d0-0242ac130003"
-	primaryIP          = "10.0.0.1"
-	overlayPrimaryIP   = "10.0.0.1/30"
-	subnetAddressSpace = "10.0.0.0/24"
-	subnetName         = "subnet1"
-	subnetPrefixLen    = 24
-	testSecIP          = "10.0.0.2"
-	version            = 1
-	nodeIP             = "10.1.0.5"
+	uuid                        = "539970a2-c2dd-11ea-b3de-0242ac130004"
+	defaultGateway              = "10.0.0.2"
+	ipIsCIDR                    = "10.0.0.1/32"
+	ipMalformed                 = "10.0.0.0.0"
+	ncID                        = "160005ba-cd02-11ea-87d0-0242ac130003"
+	primaryIP                   = "10.0.0.1"
+	overlayPrimaryIP            = "10.0.0.1/30"
+	subnetAddressSpace          = "10.0.0.0/24"
+	subnetName                  = "subnet1"
+	subnetPrefixLen             = 24
+	testSecIP                   = "10.0.0.2"
+	version                     = 1
+	nodeIP                      = "10.1.0.5"
+	vnetBlockPrimaryIP          = "10.224.0.4"
+	vnetBlockPrimaryIPPrefix    = "10.224.0.4/30"
+	vnetBlockSubnetAddressSpace = "10.224.0.0/14"
+	vnetBlockSubnetPrefixLen    = 14
+	vnetBlockNodeIP             = "10.228.0.6"
+	vnetBlockDefaultGateway     = "10.224.0.1"
+	vnetBlockCIDR1              = "10.224.0.8/30"
+	vnetBlockCIDR2              = "10.224.0.12/30"
 )
 
 var invalidStatusMultiNC = v1alpha.NodeNetworkConfigStatus{
@@ -87,6 +95,88 @@ var validOverlayNC = v1alpha.NetworkContainer{
 	Version:            version,
 }
 
+var validVNETBlockNC = v1alpha.NetworkContainer{
+	ID:             ncID,
+	AssignmentMode: v1alpha.Static,
+	Type:           v1alpha.VNETBlock,
+	IPAssignments: []v1alpha.IPAssignment{
+		{
+			Name: uuid,
+			IP:   vnetBlockCIDR1,
+		},
+		{
+			Name: uuid,
+			IP:   vnetBlockCIDR2,
+		},
+	},
+	NodeIP:             vnetBlockNodeIP,
+	PrimaryIP:          vnetBlockPrimaryIPPrefix,
+	SubnetName:         subnetName,
+	SubnetAddressSpace: vnetBlockSubnetAddressSpace,
+	DefaultGateway:     vnetBlockDefaultGateway,
+	Version:            version,
+}
+
+var validVNETBlockRequest = &cns.CreateNetworkContainerRequest{
+	Version: strconv.FormatInt(version, 10),
+	IPConfiguration: cns.IPConfiguration{
+		GatewayIPAddress: vnetBlockDefaultGateway,
+		IPSubnet: cns.IPSubnet{
+			PrefixLength: uint8(vnetBlockSubnetPrefixLen),
+			IPAddress:    vnetBlockPrimaryIP,
+		},
+	},
+	NetworkContainerid:   ncID,
+	NetworkContainerType: cns.Docker,
+	// Ignore first IP in first CIDR Block, i.e. 10.224.0.4
+	SecondaryIPConfigs: map[string]cns.SecondaryIPConfig{
+		"10.224.0.5": {
+			IPAddress: "10.224.0.5",
+			NCVersion: version,
+		},
+		"10.224.0.6": {
+			IPAddress: "10.224.0.6",
+			NCVersion: version,
+		},
+		"10.224.0.7": {
+			IPAddress: "10.224.0.7",
+			NCVersion: version,
+		},
+		"10.224.0.8": {
+			IPAddress: "10.224.0.8",
+			NCVersion: version,
+		},
+		"10.224.0.9": {
+			IPAddress: "10.224.0.9",
+			NCVersion: version,
+		},
+		"10.224.0.10": {
+			IPAddress: "10.224.0.10",
+			NCVersion: version,
+		},
+		"10.224.0.11": {
+			IPAddress: "10.224.0.11",
+			NCVersion: version,
+		},
+		"10.224.0.12": {
+			IPAddress: "10.224.0.12",
+			NCVersion: version,
+		},
+		"10.224.0.13": {
+			IPAddress: "10.224.0.13",
+			NCVersion: version,
+		},
+		"10.224.0.14": {
+			IPAddress: "10.224.0.14",
+			NCVersion: version,
+		},
+		"10.224.0.15": {
+			IPAddress: "10.224.0.15",
+			NCVersion: version,
+		},
+	},
+}
+
 func TestCreateNCRequestFromDynamicNC(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -270,6 +360,41 @@ func TestCreateNCRequestFromStaticNC(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		// VNET Block test cases
+		{
+			name:    "valid VNET Block",
+			input:   validVNETBlockNC,
+			wantErr: false,
+			want:    validVNETBlockRequest,
+		},
+		{
+			name: "PrimaryIP is not CIDR",
+			input: v1alpha.NetworkContainer{
+				AssignmentMode:     v1alpha.Static,
+				Type:               v1alpha.VNETBlock,
+				PrimaryIP:          vnetBlockPrimaryIP,
+				ID:                 ncID,
+				SubnetAddressSpace: "10.224.0.0/14",
+			},
+			wantErr: true,
+		},
+		{
+			name: "IP assignment is not CIDR",
+			input: v1alpha.NetworkContainer{
+				AssignmentMode: v1alpha.Static,
+				Type:           v1alpha.VNETBlock,
+				PrimaryIP:      vnetBlockPrimaryIPPrefix,
+				ID:             ncID,
+				IPAssignments: []v1alpha.IPAssignment{
+					{
+						Name: uuid,
+						IP:   "10.224.0.4",
+					},
+				},
+				SubnetAddressSpace: "10.224.0.0/14",
+			},
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		tt := tt

cns/kubecontroller/nodenetworkconfig/conversion_windows.go

@@ -6,22 +6,27 @@ import (
 
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
+	"github.com/pkg/errors"
 )
 
 // createNCRequestFromStaticNCHelper generates a CreateNetworkContainerRequest from a static NetworkContainer.
 // If the NC's DefaultGateway is empty and nc type is overlay, it will set the 2nd IP (*.1) as the gateway IP and all remaining IPs as
 // secondary IPs. If the gateway is not empty, it will not reserve the 2nd IP and add it as a secondary IP.
 //
 //nolint:gocritic //ignore hugeparam
-func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) *cns.CreateNetworkContainerRequest {
+func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) (*cns.CreateNetworkContainerRequest, error) {
 	secondaryIPConfigs := map[string]cns.SecondaryIPConfig{}
-	// the masked address is the 0th IP in the subnet and startingAddr is the 2nd IP (*.1)
-	startingAddr := primaryIPPrefix.Masked().Addr().Next()
-	lastAddr := startingAddr
-	// if NC DefaultGateway is empty, set the 2nd IP (*.1) to the gateway and add the rest of the IPs as secondary IPs
-	if nc.DefaultGateway == "" {
+
+	// if NC DefaultGateway is empty, set the 0th IP to the gateway and add the rest of the IPs
+	// as secondary IPs
+	startingAddr := primaryIPPrefix.Masked().Addr() // the masked address is the 0th IP in the subnet
+	if nc.DefaultGateway == "" && nc.Type == v1alpha.Overlay {
+		// assign 0th IP to the default gateway
 		nc.DefaultGateway = startingAddr.String()
 		startingAddr = startingAddr.Next()
+	} else if nc.Type == v1alpha.VNETBlock {
+		// skipping 0th IP for the Primary IP of NC
+		startingAddr = startingAddr.Next()
 	}
 
 	// iterate through all IP addresses in the subnet described by primaryPrefix and
@@ -33,7 +38,25 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 		}
 		lastAddr = addr
 	}
-	delete(secondaryIPConfigs, lastAddr.String())
+
+	if nc.Type == v1alpha.VNETBlock {
+		// Add IPs from CIDR block to the secondary IPConfigs
+		for _, ipAssignment := range nc.IPAssignments {
+			cidrPrefix, err := netip.ParsePrefix(ipAssignment.IP)
+			if err != nil {
+				return nil, errors.Wrapf(err, "invalid CIDR block: %s", ipAssignment.IP)
+			}
+
+			// iterate through all IP addresses in the CIDR block described by cidrPrefix and
+			// add them to the request as secondary IPConfigs.
+			for addr := cidrPrefix.Masked().Addr(); cidrPrefix.Contains(addr); addr = addr.Next() {
+				secondaryIPConfigs[addr.String()] = cns.SecondaryIPConfig{
+					IPAddress: addr.String(),
+					NCVersion: int(nc.Version),
+				}
+			}
+		}
+	}
 
 	return &cns.CreateNetworkContainerRequest{
 		SecondaryIPConfigs:   secondaryIPConfigs,
@@ -44,6 +67,5 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
-		NCStatus: nc.Status,
-	}
+	}, nil
 }

cns/kubecontroller/nodenetworkconfig/reconciler.go

@@ -99,8 +99,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
 		var req *cns.CreateNetworkContainerRequest
 		var err error
 		switch nnc.Status.NetworkContainers[i].AssignmentMode { //nolint:exhaustive // skipping dynamic case
+		// For Overlay and Vnet Scale Scenarios
 		case v1alpha.Static:
 			req, err = CreateNCRequestFromStaticNC(nnc.Status.NetworkContainers[i])
+		// For Pod Subnet scenario
 		default: // For backward compatibility, default will be treated as Dynamic too.
 			req, err = CreateNCRequestFromDynamicNC(nnc.Status.NetworkContainers[i])
 			// in dynamic, we will also push this NNC to the IPAM Pool Monitor when we're done.

crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go

@@ -102,8 +102,9 @@ const (
 type NCType string
 
 const (
-	VNET    NCType = "vnet"
-	Overlay NCType = "overlay"
+	VNET      NCType = "vnet"
+	VNETBlock NCType = "vnetblock"
+	Overlay   NCType = "overlay"
 )
 
 // NetworkContainer defines the structure of a Network Container as found in NetworkConfigStatus