feat: Operator E2E test to validate FeatureStore custom resource usin…

…g remote registry (feast-dev#4822) * Added new e2e test case to do the remote registry deployment. abstracted the code to do the featurestore testing. Signed-off-by: lrangine <[email protected]> * abstracted the code even further. Now the custom resource CR will execute only once for all the test cases. Signed-off-by: lrangine <[email protected]> * fixing the operator e2e test. this was commented before by mistake. Signed-off-by: lrangine <[email protected]> * increasing the remote registry deployment timeout to see if it solves the github CI timeout issue. Signed-off-by: lrangine <[email protected]> * Trying to increase the timeout and also adding the debugging actions when there is a failure. Signed-off-by: lrangine <[email protected]> * increased the go test timeout to 30m to fix the github operator e2e test action. Signed-off-by: lrangine <[email protected]> * increased the go test timeout to 30m to fix the github operator e2e test action. Signed-off-by: lrangine <[email protected]> * increased the go test timeout to 30m to fix the github operator e2e test action. Signed-off-by: lrangine <[email protected]> * incorporating the code review comments. Signed-off-by: lrangine <[email protected]> * incorporating the code review comments by using the feastRef. Signed-off-by: lrangine <[email protected]> * incorporating the code review comments by marshaling json to the go struct object rather than map. Signed-off-by: lrangine <[email protected]> * Fixing the lint error. Signed-off-by: lrangine <[email protected]> --------- Signed-off-by: lrangine <[email protected]>
opendatahub-io · Dec 10, 2024 · d558ef7 · d558ef7
1 parent 3d6bf42
commit d558ef7
Show file tree

Hide file tree

Showing 5 changed files with 230 additions and 116 deletions.
diff --git a/.github/workflows/operator-e2e-integration-tests.yml b/.github/workflows/operator-e2e-integration-tests.yml
@@ -13,6 +13,7 @@ on:
 
 jobs:
   operator-e2e-tests:
+    timeout-minutes: 40
     if:
       ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
       (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
@@ -38,7 +39,7 @@ jobs:
 
       - name: Create KIND cluster
         run: |
-          kind create cluster --name $KIND_CLUSTER --wait 5m
+          kind create cluster --name $KIND_CLUSTER --wait 10m
 
       - name: Set up kubernetes context
         run: |
@@ -51,8 +52,16 @@ jobs:
           cd infra/feast-operator/
           make test-e2e
 
+      - name: Debug KIND Cluster when there is a failure
+        if: failure()
+        run: |
+          kubectl get pods --all-namespaces
+          kubectl describe nodes
+
       - name: Clean up
         if: always()
         run: |
           # Delete the KIND cluster after tests
           kind delete cluster --name kind-$KIND_CLUSTER
+
+      
diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile
@@ -117,7 +117,7 @@ test: build-installer fmt vet lint envtest ## Run tests.
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e:
-	go test ./test/e2e/ -v -ginkgo.v
+	go test -timeout 30m ./test/e2e/ -v -ginkgo.v
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter & yamllint

diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go
@@ -28,145 +28,188 @@ import (
 )
 
 const feastControllerNamespace = "feast-operator-system"
+const timeout = 2 * time.Minute
+const controllerDeploymentName = "feast-operator-controller-manager"
 
 var _ = Describe("controller", Ordered, func() {
 	BeforeAll(func() {
 		By("creating manager namespace")
 		cmd := exec.Command("kubectl", "create", "ns", feastControllerNamespace)
 		_, _ = utils.Run(cmd)
+		var err error
+		// projectimage stores the name of the image used in the example
+		var projectimage = "localhost/feast-operator:v0.0.1"
+
+		By("building the manager(Operator) image")
+		cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage))
+		_, err = utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+		By("loading the the manager(Operator) image on Kind")
+		err = utils.LoadImageToKindClusterWithName(projectimage)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+		By("building the feast image")
+		cmd = exec.Command("make", "feast-ci-dev-docker-img")
+		_, err = utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+		// this image will be built in above make target.
+		var feastImage = "feastdev/feature-server:dev"
+		var feastLocalImage = "localhost/feastdev/feature-server:dev"
+
+		By("Tag the local feast image for the integration tests")
+		cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage)
+		_, err = utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+		By("loading the the feast image on Kind cluster")
+		err = utils.LoadImageToKindClusterWithName(feastLocalImage)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+		By("installing CRDs")
+		cmd = exec.Command("make", "install")
+		_, err = utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+		By("deploying the controller-manager")
+		cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage))
+		_, err = utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+		By("Validating that the controller-manager deployment is in available state")
+		err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout)
+		Expect(err).To(BeNil(), fmt.Sprintf(
+			"Deployment %s is not available but expected to be available. \nError: %v\n",
+			controllerDeploymentName, err,
+		))
+		fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName)
 	})
 
 	AfterAll(func() {
 		//Add any post clean up code here.
+		By("Uninstalling the feast CRD")
+		cmd := exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace)
+		_, err := utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
 	})
 
-	Context("Operator", func() {
+	Context("Operator E2E Tests", func() {
 		It("Should be able to deploy and run a default feature store CR successfully", func() {
-			//var controllerPodName string
-			var err error
-
-			// projectimage stores the name of the image used in the example
-			var projectimage = "localhost/feast-operator:v0.0.1"
-
-			By("building the manager(Operator) image")
-			cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage))
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("loading the the manager(Operator) image on Kind")
-			err = utils.LoadImageToKindClusterWithName(projectimage)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("building the feast image")
-			cmd = exec.Command("make", "feast-ci-dev-docker-img")
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-			// this image will be built in above make target.
-			var feastImage = "feastdev/feature-server:dev"
-			var feastLocalImage = "localhost/feastdev/feature-server:dev"
-
-			By("Tag the local feast image for the integration tests")
-			cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage)
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("loading the the feast image on Kind cluster")
-			err = utils.LoadImageToKindClusterWithName(feastLocalImage)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("installing CRDs")
-			cmd = exec.Command("make", "install")
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("deploying the controller-manager")
-			cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage))
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			timeout := 2 * time.Minute
-
-			controllerDeploymentName := "feast-operator-controller-manager"
-			By("Validating that the controller-manager deployment is in available state")
-			err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout)
-			Expect(err).To(BeNil(), fmt.Sprintf(
-				"Deployment %s is not available but expected to be available. \nError: %v\n",
-				controllerDeploymentName, err,
-			))
-			fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName)
-
 			By("deploying the Simple Feast Custom Resource to Kubernetes")
-			cmd = exec.Command("kubectl", "apply", "-f",
-				"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml")
+			namespace := "default"
+			cmd := exec.Command("kubectl", "apply", "-f",
+				"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace)
 			_, cmdOutputerr := utils.Run(cmd)
 			ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
 
-			namespace := "default"
-
-			deploymentNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online",
-				"feast-simple-feast-setup-offline"}
-			for _, deploymentName := range deploymentNames {
-				By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName))
-				err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout)
-				Expect(err).To(BeNil(), fmt.Sprintf(
-					"Deployment %s is not available but expected to be available. \nError: %v\n",
-					deploymentName, err,
-				))
-				fmt.Printf("Feast Deployment %s is available\n", deploymentName)
-			}
-
-			By("Check if the feast client - kubernetes config map exists.")
-			configMapName := "feast-simple-feast-setup-client"
-			err = checkIfConfigMapExists(namespace, configMapName)
-			Expect(err).To(BeNil(), fmt.Sprintf(
-				"config map %s is not available but expected to be available. \nError: %v\n",
-				configMapName, err,
-			))
-			fmt.Printf("Feast Deployment %s is available\n", configMapName)
-
-			serviceAccountNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online",
-				"feast-simple-feast-setup-offline"}
-			for _, serviceAccountName := range serviceAccountNames {
-				By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName))
-				err = checkIfServiceAccountExists(namespace, serviceAccountName)
-				Expect(err).To(BeNil(), fmt.Sprintf(
-					"Service account %s does not exist in namespace %s. Error: %v",
-					serviceAccountName, namespace, err,
-				))
-				fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace)
-			}
-
-			serviceNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online",
-				"feast-simple-feast-setup-offline"}
-			for _, serviceName := range serviceNames {
-				By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName))
-				err = checkIfKubernetesServiceExists(namespace, serviceName)
-				Expect(err).To(BeNil(), fmt.Sprintf(
-					"kubernetes service %s is not available but expected to be available. \nError: %v\n",
-					serviceName, err,
-				))
-				fmt.Printf("kubernetes service %s is available\n", serviceName)
-			}
-
-			By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", "simple-feast-setup"))
-			err = checkIfFeatureStoreCustomResourceConditionsInReady("simple-feast-setup", namespace)
-			Expect(err).To(BeNil(), fmt.Sprintf(
-				"FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n",
-				"simple-feast-setup", err,
-			))
-			fmt.Printf("FeatureStore customer resource %s conditions are in Ready State\n", "simple-feast-setup")
+			featureStoreName := "simple-feast-setup"
+			validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout)
 
 			By("deleting the feast deployment")
 			cmd = exec.Command("kubectl", "delete", "-f",
 				"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml")
 			_, cmdOutputerr = utils.Run(cmd)
 			ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
+		})
+
+		It("Should be able to deploy and run a feature store with remote registry CR successfully", func() {
+			By("deploying the Simple Feast Custom Resource to Kubernetes")
+			namespace := "default"
+			cmd := exec.Command("kubectl", "apply", "-f",
+				"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace)
+			_, cmdOutputerr := utils.Run(cmd)
+			ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
+
+			featureStoreName := "simple-feast-setup"
+			validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout)
+
+			var remoteRegistryNs = "remote-registry"
+			cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs)
+			_, _ = utils.Run(cmd)
+
+			By("deploying the Simple Feast remote registry Custom Resource to Kubernetes")
+			cmd = exec.Command("kubectl", "apply", "-f",
+				"test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs)
+			_, cmdOutputerr = utils.Run(cmd)
+			ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
 
-			By("Uninstalling the feast CRD")
-			cmd = exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace)
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			remoteFeatureStoreName := "simple-feast-remote-setup"
+
+			validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, timeout)
+
+			By("deleting the feast remote registry deployment")
+			cmd = exec.Command("kubectl", "delete", "-f",
+				"test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs)
+			_, cmdOutputerr = utils.Run(cmd)
+			ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
 
+			By("deleting the feast deployment")
+			cmd = exec.Command("kubectl", "delete", "-f",
+				"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace)
+			_, cmdOutputerr = utils.Run(cmd)
+			ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
 		})
 	})
 })
+
+func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, timeout time.Duration) {
+	hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName)
+	Expect(err).To(BeNil(), fmt.Sprintf(
+		"Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n",
+		featureStoreName, err))
+
+	k8ResourceNames := []string{fmt.Sprintf("feast-%s-online", featureStoreName),
+		fmt.Sprintf("feast-%s-offline", featureStoreName),
+	}
+
+	if !hasRemoteRegistry {
+		k8ResourceNames = append(k8ResourceNames, fmt.Sprintf("feast-%s-registry", featureStoreName))
+	}
+
+	for _, deploymentName := range k8ResourceNames {
+		By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName))
+		err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout)
+		Expect(err).To(BeNil(), fmt.Sprintf(
+			"Deployment %s is not available but expected to be available. \nError: %v\n",
+			deploymentName, err,
+		))
+		fmt.Printf("Feast Deployment %s is available\n", deploymentName)
+	}
+
+	By("Check if the feast client - kubernetes config map exists.")
+	configMapName := fmt.Sprintf("feast-%s-client", featureStoreName)
+	err = checkIfConfigMapExists(namespace, configMapName)
+	Expect(err).To(BeNil(), fmt.Sprintf(
+		"config map %s is not available but expected to be available. \nError: %v\n",
+		configMapName, err,
+	))
+	fmt.Printf("Feast Deployment client config map %s is available\n", configMapName)
+
+	for _, serviceAccountName := range k8ResourceNames {
+		By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName))
+		err = checkIfServiceAccountExists(namespace, serviceAccountName)
+		Expect(err).To(BeNil(), fmt.Sprintf(
+			"Service account %s does not exist in namespace %s. Error: %v",
+			serviceAccountName, namespace, err,
+		))
+		fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace)
+	}
+
+	for _, serviceName := range k8ResourceNames {
+		By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName))
+		err = checkIfKubernetesServiceExists(namespace, serviceName)
+		Expect(err).To(BeNil(), fmt.Sprintf(
+			"kubernetes service %s is not available but expected to be available. \nError: %v\n",
+			serviceName, err,
+		))
+		fmt.Printf("kubernetes service %s is available\n", serviceName)
+	}
+
+	By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName))
+	err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace)
+	Expect(err).To(BeNil(), fmt.Sprintf(
+		"FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n",
+		featureStoreName, err,
+	))
+	fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName)
+}