perf: [NPM] [LINUX] add NetPols in background

.github/workflows/cyclonus-netpol-extended-nightly-test.yaml

@@ -15,11 +15,10 @@ jobs:
         # run cyclonus tests in parallel for NPM with the given ConfigMaps
         profile:
           [
-            v1-default.yaml,
-            v1-place-azure-chain-first.yaml,
-            v2-default.yaml,
             v2-apply-on-need.yaml,
-            v2-place-azure-after-kube-services.yaml,
+            v2-background.yaml,
+            v2-foreground.yaml,
+            v2-place-first.yaml,
           ]
     steps:
       - name: Checkout

.github/workflows/cyclonus-netpol-test.yaml

@@ -20,7 +20,13 @@ jobs:
     strategy:
       matrix:
         # run cyclonus tests in parallel for NPM with the given ConfigMaps
-        profile: [v1-default.yaml, v1-place-azure-chain-first.yaml, v2-default.yaml, v2-apply-on-need.yaml, v2-place-azure-after-kube-services.yaml]
+        profile:
+          [
+            v2-apply-on-need.yaml,
+            v2-background.yaml,
+            v2-foreground.yaml,
+            v2-place-first.yaml,
+          ]
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.pipelines/npm/npm-conformance-tests.yaml

@@ -90,21 +90,21 @@ jobs:
     displayName: "Run Kubernetes Network Policy Test Suite"
     strategy:
       matrix:
-        v1-default:
-          AZURE_CLUSTER: "conformance-v1-default"
-          PROFILE: "v1-default"
+        v2-foreground:
+          AZURE_CLUSTER: "conformance-v2-foreground"
+          PROFILE: "v2-foreground"
           IS_STRESS_TEST: "false"
-        v2-default:
-          AZURE_CLUSTER: "conformance-v2-default"
-          PROFILE: "v2-default"
+        v2-background:
+          AZURE_CLUSTER: "conformance-v2-background"
+          PROFILE: "v2-background"
           IS_STRESS_TEST: "false"
-        v2-default-ws22:
-          AZURE_CLUSTER: "conformance-v2-default-ws22"
+        v2-ws22:
+          AZURE_CLUSTER: "conformance-v2-ws22"
           PROFILE: "v2-default-ws22"
           IS_STRESS_TEST: "false"
-        v2-default-stress:
-          AZURE_CLUSTER: "conformance-v2-default-stress"
-          PROFILE: "v2-default"
+        v2-linux-stress:
+          AZURE_CLUSTER: "conformance-v2-linux-stress"
+          PROFILE: "v2-background"
           IS_STRESS_TEST: "true"
     pool:
       name: $(BUILD_POOL_NAME_DEFAULT)
@@ -117,7 +117,7 @@ jobs:
       TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
       FQDN: empty
     steps:
-      - checkout: none
+      - checkout: self
       - download: current
         artifact: Test
 
@@ -200,7 +200,7 @@ jobs:
               fi
 
               az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig
-              ./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/examples/windows/azure-npm.yaml
+              ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/examples/windows/azure-npm.yaml
               ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-ltsc2022-$(TAG)
 
             else
@@ -219,13 +219,13 @@ jobs:
               az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig
 
               # deploy azure-npm
-              ./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/azure-npm.yaml
+              ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/azure-npm.yaml
 
               # swap azure-npm image with one built during run
               ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:linux-amd64-$(TAG)
 
               # swap NPM profile with one specified as parameter
-              ./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/profiles/$(PROFILE).yaml
+              ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/profiles/$(PROFILE).yaml
               ./kubectl --kubeconfig=./kubeconfig rollout restart ds azure-npm -n kube-system
             fi
 
@@ -437,7 +437,7 @@ jobs:
             chmod +x kubectl
 
             # deploy azure-npm
-            ./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/examples/windows/azure-npm.yaml
+            ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/examples/windows/azure-npm.yaml
 
             # swap azure-npm image with one built during run
             ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-ltsc2022-$(TAG)

.pipelines/npm/npm-scale-test.yaml

@@ -78,10 +78,10 @@ jobs:
       FQDN: empty
     strategy:
       matrix:
-        # v2-linux:
-        #   PROFILE: "sc-lin"
-        #   NUM_NETPOLS: 800
-        #   INITIAL_CONNECTIVITY_TIMEOUT: 60
+        v2-linux:
+          PROFILE: "sc-lin"
+          NUM_NETPOLS: 800
+          INITIAL_CONNECTIVITY_TIMEOUT: 60
         ws22:
           PROFILE: "sc-ws22"
           NUM_NETPOLS: 50

network/hnswrapper/hnsv2wrapperfake.go

@@ -150,10 +150,12 @@ func (f Hnsv2wrapperFake) ModifyNetworkSettings(network *hcn.HostComputeNetwork,
 			if setpol.PolicyType != hcn.SetPolicyTypeIpSet && setpol.Values != "" {
 				// Check Nested SetPolicy members
 				members := strings.Split(setpol.Values, ",")
-				for _, memberID := range members {
-					_, ok := networkCache.Policies[memberID]
-					if !ok {
-						return newErrorFakeHNS(fmt.Sprintf("Member Policy %s not found for hcn.RequestTypeUpdate", memberID))
+				if setpol.Values != "" {
+					for _, memberID := range members {
+						_, ok := networkCache.Policies[memberID]
+						if !ok {
+							return newErrorFakeHNS(fmt.Sprintf("Member Policy %s not found for hcn.RequestTypeUpdate", memberID))
+						}
 					}
 				}
 			}

npm/azure-npm.yaml

@@ -148,19 +148,22 @@ metadata:
 data:
   azure-npm.json: |
     {
-        "ResyncPeriodInMinutes":       15,
-        "ListeningPort":               10091,
-        "ListeningAddress":            "0.0.0.0",
-        "ApplyMaxBatches":             100,
-        "ApplyIntervalInMilliseconds": 500,
-        "MaxBatchedACLsPerPod":        30,
+        "ResyncPeriodInMinutes":          15,
+        "ListeningPort":                  10091,
+        "ListeningAddress":               "0.0.0.0",
+        "ApplyIntervalInMilliseconds":    500,
+        "ApplyMaxBatches":                100,
+        "MaxBatchedACLsPerPod":           30,
+        "NetPolInvervalInMilliseconds":   500,
+        "MaxPendingNetPols":              100,
         "Toggles": {
             "EnablePrometheusMetrics": true,
             "EnablePprof":             true,
             "EnableHTTPDebugAPI":      true,
             "EnableV2NPM":             true,
-            "PlaceAzureChainFirst":    true,
+            "PlaceAzureChainFirst":    false,
             "ApplyIPSetsOnNeed":       false,
-            "ApplyInBackground":       true
+            "ApplyInBackground":       true,
+            "NetPolInBackground":      true
         }
     }

npm/cmd/start.go

@@ -125,6 +125,19 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error {
 		// update the dataplane config
 		npmV2DataplaneCfg.MaxBatchedACLsPerPod = config.MaxBatchedACLsPerPod
 
+		npmV2DataplaneCfg.NetPolInBackground = config.Toggles.NetPolInBackground
+		if config.NetPolInvervalInMilliseconds > 0 {
+			npmV2DataplaneCfg.NetPolInterval = time.Duration(config.NetPolInvervalInMilliseconds * int(time.Millisecond))
+		} else {
+			npmV2DataplaneCfg.NetPolInterval = time.Duration(npmconfig.DefaultConfig.NetPolInvervalInMilliseconds * int(time.Millisecond))
+		}
+
+		if config.MaxPendingNetPols > 0 {
+			npmV2DataplaneCfg.MaxPendingNetPols = config.MaxPendingNetPols
+		} else {
+			npmV2DataplaneCfg.MaxPendingNetPols = npmconfig.DefaultConfig.MaxPendingNetPols
+		}
+
 		npmV2DataplaneCfg.ApplyInBackground = config.Toggles.ApplyInBackground
 		if config.ApplyMaxBatches > 0 {
 			npmV2DataplaneCfg.ApplyMaxBatches = config.ApplyMaxBatches

npm/config/config.go

@@ -7,6 +7,8 @@ const (
 	defaultApplyMaxBatches      = 100
 	defaultApplyInterval        = 500
 	defaultMaxBatchedACLsPerPod = 30
+	defaultMaxPendingNetPols    = 100
+	defaultNetPolInterval       = 500
 	defaultListeningPort        = 10091
 	defaultGrpcPort             = 10092
 	defaultGrpcServicePort      = 9002
@@ -35,14 +37,20 @@ var DefaultConfig = Config{
 	ApplyIntervalInMilliseconds: defaultApplyInterval,
 	MaxBatchedACLsPerPod:        defaultMaxBatchedACLsPerPod,
 
+	MaxPendingNetPols:            defaultMaxPendingNetPols,
+	NetPolInvervalInMilliseconds: defaultNetPolInterval,
+
 	Toggles: Toggles{
 		EnablePrometheusMetrics: true,
 		EnablePprof:             true,
 		EnableHTTPDebugAPI:      true,
 		EnableV2NPM:             true,
-		PlaceAzureChainFirst:    util.PlaceAzureChainFirst,
+		PlaceAzureChainFirst:    util.PlaceAzureChainAfterKubeServices,
 		ApplyIPSetsOnNeed:       false,
-		ApplyInBackground:       true,
+		// ApplyInBackground is currently used in Windows to apply the following in background: IPSets and NetPols for new/updated Pods
+		ApplyInBackground: true,
+		// NetPolInBackground is currently used in Linux to apply NetPol controller Add events in the background
+		NetPolInBackground: true,
 	},
 }
 
@@ -69,8 +77,10 @@ type Config struct {
 	// MaxBatchedACLsPerPod is the maximum number of ACLs that can be added to a Pod at once in Windows.
 	// The zero value is valid.
 	// A NetworkPolicy's ACLs are always in the same batch, and there will be at least one NetworkPolicy per batch.
-	MaxBatchedACLsPerPod int     `json:"MaxBatchedACLsPerPod,omitempty"`
-	Toggles              Toggles `json:"Toggles,omitempty"`
+	MaxBatchedACLsPerPod         int     `json:"MaxBatchedACLsPerPod,omitempty"`
+	MaxPendingNetPols            int     `json:"MaxPendingNetPols,omitempty"`
+	NetPolInvervalInMilliseconds int     `json:"NetPolInvervalInMilliseconds,omitempty"`
+	Toggles                      Toggles `json:"Toggles,omitempty"`
 }
 
 type Toggles struct {
@@ -82,6 +92,8 @@ type Toggles struct {
 	ApplyIPSetsOnNeed       bool
 	// ApplyInBackground applies for Windows only
 	ApplyInBackground bool
+	// NetPolInBackground
+	NetPolInBackground bool
 }
 
 type Flags struct {

npm/examples/windows/azure-npm-capz.yaml

@@ -145,17 +145,17 @@ data:
         "ResyncPeriodInMinutes": 15,
         "ListeningPort":         10091,
         "ListeningAddress":      "0.0.0.0",
+        "ApplyIntervalInMilliseconds":    500,
+        "ApplyMaxBatches":                100,
+        "MaxBatchedACLsPerPod":           30,
         "Toggles": {
             "EnablePrometheusMetrics": true,
             "EnablePprof":             true,
             "EnableHTTPDebugAPI":      true,
             "EnableV2NPM":             true,
             "PlaceAzureChainFirst":    true,
-            "ApplyIPSetsOnNeed":       false
+            "ApplyIPSetsOnNeed":       false,
+            "ApplyInBackground":       true,
+            "NetPolInBackground":      false
         },
-        "Transport": {
-          "Address": "azure-npm.kube-system.svc.cluster.local",
-          "Port": 10092,
-          "ServicePort": 9001
-        }
     }

npm/examples/windows/azure-npm.yaml

@@ -140,19 +140,22 @@ metadata:
 data:
   azure-npm.json: |
     {
-        "ResyncPeriodInMinutes":       15,
-        "ListeningPort":               10091,
-        "ListeningAddress":            "0.0.0.0",
-        "ApplyMaxBatches":             100,
-        "ApplyIntervalInMilliseconds": 500,
-        "MaxBatchedACLsPerPod":        30,
+        "ResyncPeriodInMinutes":          15,
+        "ListeningPort":                  10091,
+        "ListeningAddress":               "0.0.0.0",
+        "ApplyIntervalInMilliseconds":    500,
+        "ApplyMaxBatches":                100,
+        "MaxBatchedACLsPerPod":           30,
+        "NetPolInvervalInMilliseconds":   500,
+        "MaxPendingNetPols":              100,
         "Toggles": {
             "EnablePrometheusMetrics": true,
             "EnablePprof":             true,
             "EnableHTTPDebugAPI":      true,
             "EnableV2NPM":             true,
-            "PlaceAzureChainFirst":    true,
+            "PlaceAzureChainFirst":    false,
             "ApplyIPSetsOnNeed":       false,
-            "ApplyInBackground":       true
+            "ApplyInBackground":       true,
+            "NetPolInBackground":      true
         }
     }

npm/metrics/acl_rules_linux.go

@@ -0,0 +1,43 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func RecordIPTablesRestoreLatency(timer *Timer, op OperationKind) {
+	labels := prometheus.Labels{
+		operationLabel: string(op),
+	}
+	itpablesRestoreLatency.With(labels).Observe(timer.timeElapsed())
+}
+
+func RecordIPTablesDeleteLatency(timer *Timer) {
+	iptablesDeleteLatency.Observe(timer.timeElapsed())
+}
+
+func IncIPTablesRestoreFailures(op OperationKind) {
+	labels := prometheus.Labels{
+		operationLabel: string(op),
+	}
+	iptablesRestoreFailures.With(labels).Inc()
+}
+
+func TotalIPTablesRestoreLatencyCalls(op OperationKind) (int, error) {
+	return histogramVecCount(itpablesRestoreLatency, prometheus.Labels{
+		operationLabel: string(op),
+	})
+}
+
+func TotalIPTablesDeleteLatencyCalls() (int, error) {
+	collector, ok := iptablesDeleteLatency.(prometheus.Collector)
+	if !ok {
+		return 0, errNotCollector
+	}
+	return histogramCount(collector)
+}
+
+func TotalIPTablesRestoreFailures(op OperationKind) (int, error) {
+	return counterValue(iptablesRestoreFailures.With(prometheus.Labels{
+		operationLabel: string(op),
+	}))
+}

npm/metrics/acl_rules_linux_test.go

@@ -0,0 +1,49 @@
+package metrics
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRecordIPTablesRestoreLatency(t *testing.T) {
+	timer := StartNewTimer()
+	time.Sleep(1 * time.Millisecond)
+	RecordIPTablesRestoreLatency(timer, UpdateOp)
+	timer = StartNewTimer()
+	time.Sleep(1 * time.Millisecond)
+	RecordIPTablesRestoreLatency(timer, CreateOp)
+
+	count, err := TotalIPTablesRestoreLatencyCalls(CreateOp)
+	require.Nil(t, err, "failed to get metric")
+	require.Equal(t, 1, count, "should have recorded create once")
+
+	count, err = TotalIPTablesRestoreLatencyCalls(UpdateOp)
+	require.Nil(t, err, "failed to get metric")
+	require.Equal(t, 1, count, "should have recorded update once")
+}
+
+func TestRecordIPTablesDeleteLatency(t *testing.T) {
+	timer := StartNewTimer()
+	time.Sleep(1 * time.Millisecond)
+	RecordIPTablesDeleteLatency(timer)
+
+	count, err := TotalIPTablesDeleteLatencyCalls()
+	require.Nil(t, err, "failed to get metric")
+	require.Equal(t, 1, count, "should have recorded create once")
+}
+
+func TestIncIPTablesRestoreFailures(t *testing.T) {
+	IncIPTablesRestoreFailures(CreateOp)
+	IncIPTablesRestoreFailures(UpdateOp)
+	IncIPTablesRestoreFailures(CreateOp)
+
+	count, err := TotalIPTablesRestoreFailures(CreateOp)
+	require.Nil(t, err, "failed to get metric")
+	require.Equal(t, 2, count, "should have failed to create twice")
+
+	count, err = TotalIPTablesRestoreFailures(UpdateOp)
+	require.Nil(t, err, "failed to get metric")
+	require.Equal(t, 1, count, "should have failed to update once")
+}