cnf ran: ztp hub templating and node delete migration

tests/cnf/ran/ztp/internal/helper/helper.go

@@ -37,6 +37,30 @@ func WaitForPolicyToExist(
 	return policy, err
 }
 
+// WaitForPolicyMessageToContainSubstring waits for the policy status message to contain the specified substring.
+func WaitForPolicyMessageToContainSubstring(client *clients.Settings, name, namespace, subString string) error {
+	glog.V(tsparams.LogLevel).Infof("Checking policy '%s' in namespace '%s' for message substring", name, namespace)
+
+	return wait.PollUntilContextTimeout(context.TODO(),
+		tsparams.ArgoCdChangeInterval, tsparams.ArgoCdChangeTimeout, true, func(ctx context.Context) (bool, error) {
+			policy, err := ocm.PullPolicy(client, name, namespace)
+			if err != nil {
+				return false, err
+			}
+
+			details := policy.Definition.Status.Details
+			if len(details) > 0 && len(details[0].History) > 0 {
+				message := details[0].History[0].Message
+
+				glog.V(tsparams.LogLevel).Infof("Checking if message '%s' contains substring '%s'", message, subString)
+
+				return strings.Contains(message, subString), nil
+			}
+
+			return false, nil
+		})
+}
+
 // WaitUntilSearchCollectorEnabled waits up to timeout until the KAC has the search collector addon enabled.
 func WaitUntilSearchCollectorEnabled(kac *ocm.KACBuilder, timeout time.Duration) error {
 	glog.V(tsparams.LogLevel).Infof(

tests/cnf/ran/ztp/internal/helper/nodedelete.go

@@ -0,0 +1,177 @@
+package helper
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/openshift-kni/eco-goinfra/pkg/assisted"
+	"github.com/openshift-kni/eco-goinfra/pkg/bmh"
+	"github.com/openshift-kni/eco-goinfra/pkg/clients"
+	"github.com/openshift-kni/eco-goinfra/pkg/nodes"
+	. "github.com/openshift-kni/eco-gotests/tests/cnf/ran/internal/raninittools"
+	"github.com/openshift-kni/eco-gotests/tests/cnf/ran/ztp/internal/tsparams"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+// IsSnoPlusOne checks if the specified cluster has one control plane and one worker node.
+func IsSnoPlusOne(client *clients.Settings) (bool, error) {
+	controlPlanes, err := listNodesByLabel(client, RANConfig.ControlPlaneLabelMap)
+	if err != nil {
+		return false, err
+	}
+
+	if len(controlPlanes) != 1 {
+		return false, nil
+	}
+
+	glog.V(tsparams.LogLevel).Info("Exactly one control plane node found")
+
+	workers, err := listNodesByLabel(client, RANConfig.WorkerLabelMap)
+	if err != nil {
+		return false, err
+	}
+
+	trueWorkers := 0
+
+	for _, worker := range workers {
+		if !isNodeControlPlane(worker) {
+			trueWorkers++
+		}
+	}
+
+	if trueWorkers != 1 {
+		return false, nil
+	}
+
+	glog.V(tsparams.LogLevel).Info("Exactly one worker node found")
+
+	return true, nil
+}
+
+// GetPlusOneWorkerName gets the name of the one worker in a SNO+1 cluster.
+func GetPlusOneWorkerName(client *clients.Settings) (string, error) {
+	workers, err := listNodesByLabel(client, RANConfig.WorkerLabelMap)
+	if err != nil {
+		return "", err
+	}
+
+	for _, worker := range workers {
+		if !isNodeControlPlane(worker) {
+			return worker.Definition.Name, nil
+		}
+	}
+
+	return "", fmt.Errorf("could not find a worker node for cluster")
+}
+
+// IsClusterStable checks if the provided cluster does not have any unschedulable nodes.
+func IsClusterStable(client *clients.Settings) (bool, error) {
+	nodeList, err := nodes.List(client)
+	if err != nil {
+		return false, err
+	}
+
+	for _, node := range nodeList {
+		if node.Definition.Spec.Unschedulable {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+// WaitForNumberOfNodes waits up to timeout until the number of nodes on the cluster matches the expected.
+func WaitForNumberOfNodes(client *clients.Settings, expected int, timeout time.Duration) error {
+	return wait.PollUntilContextTimeout(
+		context.TODO(), 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
+			nodeList, err := nodes.List(client)
+			if err != nil {
+				return false, err
+			}
+
+			if len(nodeList) == expected {
+				return true, nil
+			}
+
+			glog.V(tsparams.LogLevel).Infof("Expected %d nodes but found %d nodes", expected, len(nodeList))
+
+			return false, nil
+		})
+}
+
+// GetBmhNamespace returns the namespace for the specified BareMetalHost, if it exists.
+func GetBmhNamespace(client *clients.Settings, bmhName string) (string, error) {
+	bmhList, err := bmh.ListInAllNamespaces(client)
+	if err != nil {
+		return "", err
+	}
+
+	for _, bmhBuilder := range bmhList {
+		if bmhBuilder.Definition.Name == bmhName {
+			return bmhBuilder.Definition.Namespace, nil
+		}
+	}
+
+	return "", fmt.Errorf("BareMetalHost %s not found", bmhName)
+}
+
+// WaitForBMHDeprovisioning waits up to the specified timeout till the BMH and agent with the provided name and
+// namespace are no longer found.
+func WaitForBMHDeprovisioning(client *clients.Settings, name, namespace string, timeout time.Duration) error {
+	return wait.PollUntilContextTimeout(
+		context.TODO(), tsparams.ArgoCdChangeInterval, timeout, true, func(ctx context.Context) (bool, error) {
+			glog.V(tsparams.LogLevel).Infof("Checking if BareMetalHost %s in namespace %s is deprovisioned", name, namespace)
+
+			_, err := bmh.Pull(client, name, namespace)
+			if err == nil {
+				return false, nil
+			}
+
+			_, err = assisted.PullAgent(client, name, namespace)
+			if err == nil {
+				return false, nil
+			}
+
+			return true, nil
+		})
+}
+
+// WaitForBMHAnnotation waits up to timeout until the specified annotation is present on the BMH.
+func WaitForBMHAnnotation(bareMetalHost *bmh.BmhBuilder, annotation string, timeout time.Duration) error {
+	return wait.PollUntilContextTimeout(
+		context.TODO(), tsparams.ArgoCdChangeInterval, timeout, true, func(ctx context.Context) (bool, error) {
+			if !bareMetalHost.Exists() {
+				glog.V(tsparams.LogLevel).Infof("BMH %s in namespace %s does not exist")
+
+				return false, fmt.Errorf(
+					"bmh object %s does not exist in namespace %s", bareMetalHost.Definition.Name, bareMetalHost.Definition.Namespace)
+			}
+
+			if _, ok := bareMetalHost.Object.Annotations[annotation]; !ok {
+				return false, nil
+			}
+
+			glog.V(tsparams.LogLevel).Infof(
+				"Annotation %s found for BMH %s/%s", annotation, bareMetalHost.Definition.Name, bareMetalHost.Definition.Namespace)
+
+			return true, nil
+		})
+}
+
+// listNodesByLabel returns a list of nodes that have the specified label.
+func listNodesByLabel(client *clients.Settings, labelMap map[string]string) ([]*nodes.Builder, error) {
+	return nodes.List(client, metav1.ListOptions{
+		LabelSelector: labels.Set(labelMap).String(),
+	})
+}
+
+// isNodeControlPlane checks whether the provided node is a control plane node.
+func isNodeControlPlane(node *nodes.Builder) bool {
+	_, exists := node.Definition.Labels[RANConfig.ControlPlaneLabel]
+
+	return exists
+}

tests/cnf/ran/ztp/internal/tsparams/consts.go

@@ -13,14 +13,17 @@ const (
 	LabelArgoCdAcmCrsTestCases = "ztp-argocd-acm-crs"
 	// LabelArgoCdClustersAppTestCases is the label for the Argo CD clusters app test cases.
 	LabelArgoCdClustersAppTestCases = "ztp-argocd-clusters"
-
-	// TestNamespace is the namespace used for ZTP tests.
-	TestNamespace = "ztp-test"
+	// LabelArgoCdHubTemplatingTestCases is the label for a particular set of test cases.
+	LabelArgoCdHubTemplatingTestCases = "ztp-argocd-hub-templating"
+	// LabelArgoCdNodeDeletionTestCases is the label for a particular set of test cases.
+	LabelArgoCdNodeDeletionTestCases = "ztp-argocd-node-delete"
 
 	// MultiClusterHubOperator is the name of the multi cluster hub operator.
 	MultiClusterHubOperator = "multiclusterhub-operator"
 	// AcmPolicyGeneratorName is the name of the ACM policy generator container.
 	AcmPolicyGeneratorName = "acm-policy-generator"
+	// TalmHubPodName is the name of the TALM pod on the hub cluster.
+	TalmHubPodName = "cluster-group-upgrades-controller-manager"
 
 	// ArgoCdPoliciesAppName is the name of the policies app in Argo CD.
 	ArgoCdPoliciesAppName = "policies"
@@ -38,11 +41,32 @@ const (
 	ZtpTestPathClustersApp = "ztp-test/klusterlet-addon"
 	// ZtpTestPathRemoveNmState is the git path for the remove nm state test.
 	ZtpTestPathRemoveNmState = "ztp-test/remove-nmstate"
+	// ZtpTestPathTemplatingAutoIndent is the git path for the templating auto indent test.
+	ZtpTestPathTemplatingAutoIndent = "ztp-test/hub-templating-autoindent"
+	// ZtpTestPathTemplatingValid is the git path for the templating valid test.
+	ZtpTestPathTemplatingValid = "ztp-test/hub-templating-valid"
+	// ZtpTestPathTemplatingValid416 is the git path for the templating valid test starting from TALM 4.16.
+	ZtpTestPathTemplatingValid416 = "ztp-test/hub-templating-valid-4.16"
+	// ZtpTestPathNodeDeleteAddAnnotation is the git path for the node deletion add annotation test.
+	ZtpTestPathNodeDeleteAddAnnotation = "ztp-test/node-delete/add-annotation"
+	// ZtpTestPathNodeDeleteAddSuppression is the git path for the node deletion add suppression test.
+	ZtpTestPathNodeDeleteAddSuppression = "ztp-test/node-delete/add-suppression"
 	// ZtpKustomizationPath is the path to the kustomization file in the ztp test.
 	ZtpKustomizationPath = "/kustomization.yaml"
 
+	// TestNamespace is the namespace used for ZTP tests.
+	TestNamespace = "ztp-test"
 	// AcmCrsPolicyName is the name of the policy for ACM CRs.
 	AcmCrsPolicyName = "acm-crs-policy"
+	// HubTemplatingPolicyName is the name used for the hub templating policy.
+	HubTemplatingPolicyName = "hub-templating-policy-sriov-config"
+	// HubTemplatingCguName is the name used for the hub templating CGU.
+	HubTemplatingCguName = "hub-templating"
+	// HubTemplatingCguNamespace is the namespace used by the hub templating CGU. It should be different than the
+	// policy namespace.
+	HubTemplatingCguNamespace = "default"
+	// NodeDeletionCrAnnotation is the annotation applied in the node deletion tests.
+	NodeDeletionCrAnnotation = "bmac.agent-install.openshift.io/remove-agent-and-node-on-delete"
 
 	// LogLevel is the verbosity of glog statements in this test suite.
 	LogLevel glog.Level = 90

tests/cnf/ran/ztp/internal/tsparams/ztpvars.go

@@ -4,6 +4,7 @@ import (
 	"github.com/openshift-kni/eco-gotests/tests/cnf/ran/internal/ranparam"
 	"github.com/openshift-kni/k8sreporter"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // ArgoCdGitDetails is the details for a single app in ArgoCD.
@@ -33,4 +34,12 @@ var (
 	}
 	// ArgoCdAppDetails contains more details for each of the ArgoCdApps.
 	ArgoCdAppDetails = map[string]ArgoCdGitDetails{}
+
+	// InvalidManagedPoliciesCondition is the CGU condition for where there are invalid managed policies.
+	InvalidManagedPoliciesCondition = metav1.Condition{
+		Type:    "Validated",
+		Status:  metav1.ConditionFalse,
+		Reason:  "NotAllManagedPoliciesExist",
+		Message: "Invalid managed policies",
+	}
 )

tests/cnf/ran/ztp/tests/ztp-argocd-acm-crs.go

@@ -6,7 +6,6 @@ import (
 	"github.com/golang/glog"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"github.com/openshift-kni/eco-goinfra/pkg/clients"
 	"github.com/openshift-kni/eco-goinfra/pkg/deployment"
 	"github.com/openshift-kni/eco-goinfra/pkg/reportxml"
 	"github.com/openshift-kni/eco-gotests/tests/cnf/ran/internal/ranhelper"
@@ -24,11 +23,6 @@ var _ = Describe("ZTP Argo CD ACM CR Tests", Label(tsparams.LabelArgoCdAcmCrsTes
 	)
 
 	BeforeEach(func() {
-		By("checking that the required clusters are present")
-		if !ranhelper.AreClustersPresent([]*clients.Settings{HubAPIClient, Spoke1APIClient}) {
-			Skip("not all of the required clusters are present")
-		}
-
 		By("verifying that ZTP meets the minimum version")
 		versionInRange, err := ranhelper.IsVersionStringInRange(RANConfig.ZTPVersion, "4.12", "")
 		Expect(err).ToNot(HaveOccurred(), "Failed to compare ZTP version string")

tests/cnf/ran/ztp/tests/ztp-argocd-clusters-app.go

@@ -6,7 +6,6 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/openshift-kni/eco-goinfra/pkg/assisted"
-	"github.com/openshift-kni/eco-goinfra/pkg/clients"
 	"github.com/openshift-kni/eco-goinfra/pkg/ocm"
 	"github.com/openshift-kni/eco-goinfra/pkg/reportxml"
 	"github.com/openshift-kni/eco-gotests/tests/cnf/ran/internal/ranhelper"
@@ -17,11 +16,6 @@ import (
 
 var _ = Describe("ZTP Argo CD Clusters Tests", Label(tsparams.LabelArgoCdClustersAppTestCases), func() {
 	BeforeEach(func() {
-		By("checking that the required clusters are present")
-		if !ranhelper.AreClustersPresent([]*clients.Settings{HubAPIClient, Spoke1APIClient}) {
-			Skip("not all of the required clusters are present")
-		}
-
 		By("verifying that ZTP meets the minimum version")
 		versionInRange, err := ranhelper.IsVersionStringInRange(RANConfig.ZTPVersion, "4.11", "")
 		Expect(err).ToNot(HaveOccurred(), "Failed to compare ZTP version string")