chore: install ip-masq-agent as part of overlay cns scenarios (#2273)

Azure · Oct 5, 2023 · 7a5cb5e · 7a5cb5e
1 parent f061370
commit 7a5cb5e
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 34 deletions.
diff --git a/.pipelines/cni/singletenancy/linux-cniv2-template.yaml b/.pipelines/cni/singletenancy/linux-cniv2-template.yaml
@@ -63,13 +63,8 @@ stages:
                 addSpnToEnvironment: true
                 inlineScript: |
                   echo "Start Integration Tests on Overlay Cluster"
-                  echo "deploy ip-masq-agent for overlay"
                   make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
-                  kubectl apply -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
-                  cd test/integration/manifests/ip-masq-agent/
-                  kubectl apply -f config-custom.yaml
-                  kubectl apply -f config-reconcile.yaml
-                  cd ../../../..
+                  kubectl cluster-info
                   kubectl get po -owide -A
                   sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(cnsVersion) CNI_DROPGZ_VERSION=$(dropgzVersion) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }}
               name: "overlaye2e"

diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml
@@ -68,14 +68,7 @@ steps:
 
   - script: |
       echo "Start Integration Tests on Overlay Cluster"
-      echo "deploy ip-masq-agent for overlay"
-      kubectl apply -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
-      cd test/integration/manifests/ip-masq-agent/
-      kubectl apply -f config-custom.yaml
-      kubectl apply -f config-reconcile.yaml
-      cd ../../../..
-      kubectl get po -owide -A
-      CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version)
+      CNS=$(make cns-version)
       sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_DROPGZ_VERSION=$(dropgzVersion) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }}
     retryCountOnTaskFailure: 3
     name: "integrationTest"

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -93,13 +93,6 @@ steps:
 
   - script: |
       echo "Start Azilium E2E Tests on Overlay Cluster"
-      echo "deploy ip-masq-agent for overlay"
-      kubectl create -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
-      cd test/integration/manifests/ip-masq-agent/
-      kubectl create configmap config-custom.yaml
-      kubectl create configmap config-reconcile.yaml
-      cd ../../../..
-      kubectl get po -owide -A
       # Nightly does not build images per commit. Will use existing image.
       if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]
       then

diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml
@@ -48,11 +48,6 @@ steps:
         displayName: "Dropgz Version"
 
   - script: |
-      echo "deploy ip-masq-agent for overlay"
-      kubectl create -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
-      cd test/integration/manifests/ip-masq-agent/
-      kubectl create configmap config-custom.yaml
-      cd ../../../..
       kubectl cluster-info
       kubectl get node
       sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(dropgzVersion) INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }}

diff --git a/test/internal/kubernetes/utils_create.go b/test/internal/kubernetes/utils_create.go
@@ -44,6 +44,7 @@ type cnsDetails struct {
 	initContainerVolumeMounts []corev1.VolumeMount
 	containerVolumeMounts     []corev1.VolumeMount
 	configMapPath             string
+	installIPMasqAgent        bool
 }
 
 const (
@@ -181,6 +182,31 @@ func MustCreateNamespace(ctx context.Context, clienset *kubernetes.Clientset, na
 	}
 }
 
+func InstallIPMasqAgent(ctx context.Context, clientset *kubernetes.Clientset) error {
+	manifestDir, err := getManifestFolder()
+	if err != nil {
+		return errors.Wrap(err, "failed to get manifest folder")
+	}
+
+	ipMasqAgentDir := path.Join(manifestDir, "/ip-masq-agent")
+	customConfigPath := path.Join(ipMasqAgentDir, "/config-custom.yaml")
+	reconcileConfigPath := path.Join(ipMasqAgentDir, "/config-reconcile.yaml")
+	daemonsetPath := path.Join(ipMasqAgentDir, "/ip-masq-agent.yaml")
+
+	MustSetupConfigMap(ctx, clientset, customConfigPath)
+	MustSetupConfigMap(ctx, clientset, reconcileConfigPath)
+
+	ds := MustParseDaemonSet(daemonsetPath)
+	dsClient := clientset.AppsV1().DaemonSets(ds.Namespace)
+	MustCreateDaemonset(ctx, dsClient, ds)
+
+	if err := WaitForPodDaemonset(ctx, clientset, ds.Namespace, ds.Name, "k8s-app=azure-ip-masq-agent-user"); err != nil {
+		return errors.Wrap(err, "failed to check daemonset running")
+	}
+
+	return nil
+}
+
 func InstallCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, logDir string) (func() error, error) {
 	cniDropgzVersion := os.Getenv(envCNIDropgzVersion)
 	cnsVersion := os.Getenv(envCNSVersion)
@@ -257,14 +283,24 @@ func RestartCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset) e
 	return nil
 }
 
-func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) {
+func getManifestFolder() (string, error) {
 	_, b, _, ok := runtime.Caller(0)
 	if !ok {
-		return map[CNSScenario]map[corev1.OSName]cnsDetails{}, errors.Wrap(ErrPathNotFound, "could not get path to caller")
+		return "", errors.Wrap(ErrPathNotFound, "could not get path to caller")
 	}
 	basepath := filepath.Dir(b)
-	cnsManifestFolder := path.Join(basepath, "../../integration/manifests/cns")
-	cnsConfigFolder := path.Join(basepath, "../../integration/manifests/cnsconfig")
+	manifestFolder := path.Join(basepath, "../../integration/manifests")
+	return manifestFolder, nil
+}
+
+func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) {
+	manifestDir, err := getManifestFolder()
+	if err != nil {
+		return map[CNSScenario]map[corev1.OSName]cnsDetails{}, errors.Wrap(err, "failed to get manifest folder")
+	}
+
+	cnsManifestFolder := path.Join(manifestDir, "/cns")
+	cnsConfigFolder := path.Join(manifestDir, "/cnsconfig")
 
 	// relative cns manifest paths
 	cnsLinuxDaemonSetPath := cnsManifestFolder + "/daemonset-linux.yaml"
@@ -297,7 +333,8 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error)
 					"-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-vnet-ipam", "-o", "/opt/cni/bin/azure-vnet-ipam",
 					"azure-swift.conflist", "-o", "/etc/cni/net.d/10-azure.conflist",
 				},
-				configMapPath: cnsSwiftConfigMapPath,
+				configMapPath:      cnsSwiftConfigMapPath,
+				installIPMasqAgent: false,
 			},
 		},
 		EnvInstallAzilium: {
@@ -312,7 +349,8 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error)
 				initContainerArgs: []string{
 					"deploy", "azure-ipam", "-o", "/opt/cni/bin/azure-ipam",
 				},
-				configMapPath: cnsCiliumConfigMapPath,
+				configMapPath:      cnsCiliumConfigMapPath,
+				installIPMasqAgent: false,
 			},
 		},
 		EnvInstallOverlay: {
@@ -327,7 +365,8 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error)
 				initContainerArgs: []string{
 					"deploy", "azure-ipam", "-o", "/opt/cni/bin/azure-ipam",
 				},
-				configMapPath: cnsOverlayConfigMapPath,
+				configMapPath:      cnsOverlayConfigMapPath,
+				installIPMasqAgent: true,
 			},
 		},
 		EnvInstallAzureCNIOverlay: {
@@ -346,6 +385,7 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error)
 				initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayLinux(),
 				containerVolumeMounts:     cnsVolumeMountsForAzureCNIOverlayLinux(),
 				configMapPath:             cnsAzureCNIOverlayLinuxConfigMapPath,
+				installIPMasqAgent:        true,
 			},
 			corev1.Windows: {
 				daemonsetPath:          cnsWindowsDaemonSetPath,
@@ -362,6 +402,7 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error)
 				initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayWindows(),
 				containerVolumeMounts:     cnsVolumeMountsForAzureCNIOverlayWindows(),
 				configMapPath:             cnsAzureCNIOverlayWindowsConfigMapPath,
+				installIPMasqAgent:        true,
 			},
 		},
 		EnvInstallDualStackOverlay: {
@@ -378,7 +419,8 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error)
 					"azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-vnet-ipam", "-o",
 					"/opt/cni/bin/azure-vnet-ipam", "azure-swift-overlay-dualstack.conflist", "-o", "/etc/cni/net.d/10-azure.conflist",
 				},
-				configMapPath: cnsSwiftConfigMapPath,
+				configMapPath:      cnsSwiftConfigMapPath,
+				installIPMasqAgent: true,
 			},
 			corev1.Windows: {
 				daemonsetPath:          cnsWindowsDaemonSetPath,
@@ -395,6 +437,7 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error)
 				initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayWindows(),
 				containerVolumeMounts:     cnsVolumeMountsForAzureCNIOverlayWindows(),
 				configMapPath:             cnsAzureCNIDualStackWindowsConfigMapPath,
+				installIPMasqAgent:        true,
 			},
 		},
 	}
@@ -429,6 +472,13 @@ func setupCNSDaemonset(
 		return appsv1.DaemonSet{}, cnsDetails{}, errors.Wrap(err, "failed to parse cns daemonset")
 	}
 
+	if cnsScenarioDetails.installIPMasqAgent {
+		log.Printf("Installing IP Masq Agent")
+		if err := InstallIPMasqAgent(ctx, clientset); err != nil {
+			return appsv1.DaemonSet{}, cnsDetails{}, errors.Wrap(err, "failed to install ip masq agent")
+		}
+	}
+
 	log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image)
 	cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace)