diff --git a/.pipelines/cni/singletenancy/linux-cniv2-template.yaml b/.pipelines/cni/singletenancy/linux-cniv2-template.yaml index 8f58fd8be4..b8945076d8 100644 --- a/.pipelines/cni/singletenancy/linux-cniv2-template.yaml +++ b/.pipelines/cni/singletenancy/linux-cniv2-template.yaml @@ -63,13 +63,8 @@ stages: addSpnToEnvironment: true inlineScript: | echo "Start Integration Tests on Overlay Cluster" - echo "deploy ip-masq-agent for overlay" make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) - kubectl apply -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false - cd test/integration/manifests/ip-masq-agent/ - kubectl apply -f config-custom.yaml - kubectl apply -f config-reconcile.yaml - cd ../../../.. + kubectl cluster-info kubectl get po -owide -A sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(cnsVersion) CNI_DROPGZ_VERSION=$(dropgzVersion) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }} name: "overlaye2e" diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml index 1c4329d8a3..e2e6c69c10 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml @@ -68,14 +68,7 @@ steps: - script: | echo "Start Integration Tests on Overlay Cluster" - echo "deploy ip-masq-agent for overlay" - kubectl apply -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false - cd test/integration/manifests/ip-masq-agent/ - kubectl apply -f config-custom.yaml - kubectl apply -f config-reconcile.yaml - cd ../../../.. - kubectl get po -owide -A - CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version) + CNS=$(make cns-version) sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_DROPGZ_VERSION=$(dropgzVersion) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }} retryCountOnTaskFailure: 3 name: "integrationTest" diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 5c829f7aca..8731f6ed5f 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -93,13 +93,6 @@ steps: - script: | echo "Start Azilium E2E Tests on Overlay Cluster" - echo "deploy ip-masq-agent for overlay" - kubectl create -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false - cd test/integration/manifests/ip-masq-agent/ - kubectl create configmap config-custom.yaml - kubectl create configmap config-reconcile.yaml - cd ../../../.. - kubectl get po -owide -A # Nightly does not build images per commit. Will use existing image. if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ] then diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index afc7542775..f4239ac67d 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -48,11 +48,6 @@ steps: displayName: "Dropgz Version" - script: | - echo "deploy ip-masq-agent for overlay" - kubectl create -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false - cd test/integration/manifests/ip-masq-agent/ - kubectl create configmap config-custom.yaml - cd ../../../.. kubectl cluster-info kubectl get node sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(dropgzVersion) INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }} diff --git a/test/internal/kubernetes/utils_create.go b/test/internal/kubernetes/utils_create.go index cf62fbe75b..9739b3bac8 100644 --- a/test/internal/kubernetes/utils_create.go +++ b/test/internal/kubernetes/utils_create.go @@ -44,6 +44,7 @@ type cnsDetails struct { initContainerVolumeMounts []corev1.VolumeMount containerVolumeMounts []corev1.VolumeMount configMapPath string + installIPMasqAgent bool } const ( @@ -181,6 +182,31 @@ func MustCreateNamespace(ctx context.Context, clienset *kubernetes.Clientset, na } } +func InstallIPMasqAgent(ctx context.Context, clientset *kubernetes.Clientset) error { + manifestDir, err := getManifestFolder() + if err != nil { + return errors.Wrap(err, "failed to get manifest folder") + } + + ipMasqAgentDir := path.Join(manifestDir, "/ip-masq-agent") + customConfigPath := path.Join(ipMasqAgentDir, "/config-custom.yaml") + reconcileConfigPath := path.Join(ipMasqAgentDir, "/config-reconcile.yaml") + daemonsetPath := path.Join(ipMasqAgentDir, "/ip-masq-agent.yaml") + + MustSetupConfigMap(ctx, clientset, customConfigPath) + MustSetupConfigMap(ctx, clientset, reconcileConfigPath) + + ds := MustParseDaemonSet(daemonsetPath) + dsClient := clientset.AppsV1().DaemonSets(ds.Namespace) + MustCreateDaemonset(ctx, dsClient, ds) + + if err := WaitForPodDaemonset(ctx, clientset, ds.Namespace, ds.Name, "k8s-app=azure-ip-masq-agent-user"); err != nil { + return errors.Wrap(err, "failed to check daemonset running") + } + + return nil +} + func InstallCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, logDir string) (func() error, error) { cniDropgzVersion := os.Getenv(envCNIDropgzVersion) cnsVersion := os.Getenv(envCNSVersion) @@ -257,14 +283,24 @@ func RestartCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset) e return nil } -func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) { +func getManifestFolder() (string, error) { _, b, _, ok := runtime.Caller(0) if !ok { - return map[CNSScenario]map[corev1.OSName]cnsDetails{}, errors.Wrap(ErrPathNotFound, "could not get path to caller") + return "", errors.Wrap(ErrPathNotFound, "could not get path to caller") } basepath := filepath.Dir(b) - cnsManifestFolder := path.Join(basepath, "../../integration/manifests/cns") - cnsConfigFolder := path.Join(basepath, "../../integration/manifests/cnsconfig") + manifestFolder := path.Join(basepath, "../../integration/manifests") + return manifestFolder, nil +} + +func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) { + manifestDir, err := getManifestFolder() + if err != nil { + return map[CNSScenario]map[corev1.OSName]cnsDetails{}, errors.Wrap(err, "failed to get manifest folder") + } + + cnsManifestFolder := path.Join(manifestDir, "/cns") + cnsConfigFolder := path.Join(manifestDir, "/cnsconfig") // relative cns manifest paths cnsLinuxDaemonSetPath := cnsManifestFolder + "/daemonset-linux.yaml" @@ -297,7 +333,8 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) "-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-vnet-ipam", "-o", "/opt/cni/bin/azure-vnet-ipam", "azure-swift.conflist", "-o", "/etc/cni/net.d/10-azure.conflist", }, - configMapPath: cnsSwiftConfigMapPath, + configMapPath: cnsSwiftConfigMapPath, + installIPMasqAgent: false, }, }, EnvInstallAzilium: { @@ -312,7 +349,8 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) initContainerArgs: []string{ "deploy", "azure-ipam", "-o", "/opt/cni/bin/azure-ipam", }, - configMapPath: cnsCiliumConfigMapPath, + configMapPath: cnsCiliumConfigMapPath, + installIPMasqAgent: false, }, }, EnvInstallOverlay: { @@ -327,7 +365,8 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) initContainerArgs: []string{ "deploy", "azure-ipam", "-o", "/opt/cni/bin/azure-ipam", }, - configMapPath: cnsOverlayConfigMapPath, + configMapPath: cnsOverlayConfigMapPath, + installIPMasqAgent: true, }, }, EnvInstallAzureCNIOverlay: { @@ -346,6 +385,7 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayLinux(), containerVolumeMounts: cnsVolumeMountsForAzureCNIOverlayLinux(), configMapPath: cnsAzureCNIOverlayLinuxConfigMapPath, + installIPMasqAgent: true, }, corev1.Windows: { daemonsetPath: cnsWindowsDaemonSetPath, @@ -362,6 +402,7 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayWindows(), containerVolumeMounts: cnsVolumeMountsForAzureCNIOverlayWindows(), configMapPath: cnsAzureCNIOverlayWindowsConfigMapPath, + installIPMasqAgent: true, }, }, EnvInstallDualStackOverlay: { @@ -378,7 +419,8 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) "azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-vnet-ipam", "-o", "/opt/cni/bin/azure-vnet-ipam", "azure-swift-overlay-dualstack.conflist", "-o", "/etc/cni/net.d/10-azure.conflist", }, - configMapPath: cnsSwiftConfigMapPath, + configMapPath: cnsSwiftConfigMapPath, + installIPMasqAgent: true, }, corev1.Windows: { daemonsetPath: cnsWindowsDaemonSetPath, @@ -395,6 +437,7 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayWindows(), containerVolumeMounts: cnsVolumeMountsForAzureCNIOverlayWindows(), configMapPath: cnsAzureCNIDualStackWindowsConfigMapPath, + installIPMasqAgent: true, }, }, } @@ -429,6 +472,13 @@ func setupCNSDaemonset( return appsv1.DaemonSet{}, cnsDetails{}, errors.Wrap(err, "failed to parse cns daemonset") } + if cnsScenarioDetails.installIPMasqAgent { + log.Printf("Installing IP Masq Agent") + if err := InstallIPMasqAgent(ctx, clientset); err != nil { + return appsv1.DaemonSet{}, cnsDetails{}, errors.Wrap(err, "failed to install ip masq agent") + } + } + log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image) cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace)