cnf ran: ztp acm crs and clusters app tests (#81)

This PR adds the first two tests cases for the ZTP functional test suite along with the helpers they require. Additionally, the OCP versions of each cluster are now logged.

tests/cnf/ran/internal/ranconfig/config.go

@@ -117,6 +117,8 @@ func (ranconfig *RANConfig) newHubConfig(configFile string) {
 		glog.V(ranparam.LogLevel).Infof("Failed to get OCP version from hub: %v", err)
 	}
 
+	glog.V(ranparam.LogLevel).Infof("Found OCP version on hub: %s", ranconfig.HubConfig.HubOCPVersion)
+
 	ranconfig.HubConfig.HubOperatorVersions = make(map[ranparam.HubOperatorName]string)
 
 	ranconfig.HubConfig.HubOperatorVersions[ranparam.ACM], err = ranhelper.GetOperatorVersionFromCsv(
@@ -182,6 +184,8 @@ func (ranconfig *RANConfig) newSpoke1Config(configFile string) {
 		glog.V(ranparam.LogLevel).Infof("Failed to get OCP version from spoke 1: %v", err)
 	}
 
+	glog.V(ranparam.LogLevel).Infof("Found OCP version on spoke 1: %s", ranconfig.Spoke1Config.Spoke1OCPVersion)
+
 	if len(ranconfig.Spoke1Config.BMCHosts) > 0 &&
 		ranconfig.Spoke1Config.BMCUsername != "" &&
 		ranconfig.Spoke1Config.BMCPassword != "" {
@@ -224,6 +228,8 @@ func (ranconfig *RANConfig) newSpoke2Config(configFile string) {
 	if err != nil {
 		glog.V(ranparam.LogLevel).Infof("Failed to get OCP version from spoke 2: %v", err)
 	}
+
+	glog.V(ranparam.LogLevel).Infof("Found OCP version on spoke 2: %s", ranconfig.Spoke2Config.Spoke2OCPVersion)
 }
 
 func readConfig[C any](config *C, configFile string) error {

tests/cnf/ran/internal/ranhelper/ranhelper.go

@@ -2,6 +2,7 @@ package ranhelper
 
 import (
 	"github.com/golang/glog"
+	"github.com/openshift-kni/eco-goinfra/pkg/clients"
 	"github.com/openshift-kni/eco-goinfra/pkg/pod"
 	"github.com/openshift-kni/eco-gotests/tests/cnf/ran/internal/ranparam"
 	v1 "k8s.io/api/core/v1"
@@ -43,6 +44,17 @@ func DoesContainerExistInPod(pod *pod.Builder, containerName string) bool {
 	return false
 }
 
+// AreClustersPresent checks all of the provided clusters and returns false if any are nil.
+func AreClustersPresent(clusters []*clients.Settings) bool {
+	for _, cluster := range clusters {
+		if cluster == nil {
+			return false
+		}
+	}
+
+	return true
+}
+
 // UnmarshalRaw converts raw bytes for a K8s CR into the actual type.
 func UnmarshalRaw[T any](raw []byte) (*T, error) {
 	untyped := &unstructured.Unstructured{}

tests/cnf/ran/ztp/internal/helper/gitdetails.go

@@ -0,0 +1,191 @@
+package helper
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/openshift-kni/eco-goinfra/pkg/argocd"
+	. "github.com/openshift-kni/eco-gotests/tests/cnf/ran/internal/raninittools"
+	"github.com/openshift-kni/eco-gotests/tests/cnf/ran/internal/ranparam"
+	"github.com/openshift-kni/eco-gotests/tests/cnf/ran/ztp/internal/tsparams"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+// GetArgoCdAppGitDetails initializes tsparams.ArgoCdAppDetails with the details for each of tsparams.ArgoCdApps.
+func GetArgoCdAppGitDetails() error {
+	for _, app := range tsparams.ArgoCdApps {
+		argoCdApp, err := argocd.PullApplication(HubAPIClient, app, ranparam.OpenshiftGitOpsNamespace)
+		if err != nil {
+			return err
+		}
+
+		tsparams.ArgoCdAppDetails[app] = tsparams.ArgoCdGitDetails{
+			Repo:   argoCdApp.Definition.Spec.Source.RepoURL,
+			Branch: argoCdApp.Definition.Spec.Source.TargetRevision,
+			Path:   argoCdApp.Definition.Spec.Source.Path,
+		}
+	}
+
+	return nil
+}
+
+// SetGitDetailsInArgoCd updates the git details for the provided Argo CD app.
+func SetGitDetailsInArgoCd(
+	appName string, gitDetails tsparams.ArgoCdGitDetails, waitForSync, syncMustBeValid bool) error {
+	app, err := argocd.PullApplication(HubAPIClient, appName, ranparam.OpenshiftGitOpsNamespace)
+	if err != nil {
+		return err
+	}
+
+	appSource := app.Definition.Spec.Source
+	if appSource.RepoURL == gitDetails.Repo &&
+		appSource.TargetRevision == gitDetails.Branch &&
+		appSource.Path == gitDetails.Path {
+		glog.V(tsparams.LogLevel).Info("Provided git details already configured, no change required.")
+
+		return nil
+	}
+
+	glog.V(tsparams.LogLevel).Infof("Updating argocd app %s to use git details %v", appName, gitDetails)
+
+	appSource.RepoURL = gitDetails.Repo
+	appSource.TargetRevision = gitDetails.Branch
+	appSource.Path = gitDetails.Path
+
+	_, err = app.Update(true)
+	if err != nil {
+		return err
+	}
+
+	if waitForSync {
+		err = waitForArgoCdChangeToComplete(appName, syncMustBeValid, tsparams.ArgoCdChangeTimeout)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// UpdateArgoCdAppGitPath updates the git path in the specified Argo CD app, returning whether the git path exists and
+// any error that occurred.
+func UpdateArgoCdAppGitPath(appName, ztpTestPath string, syncMustBeValid bool) (bool, error) {
+	gitDetails := tsparams.ArgoCdAppDetails[appName]
+	testGitPath := JoinGitPaths([]string{
+		gitDetails.Path,
+		ztpTestPath,
+	})
+
+	if !DoesGitPathExist(gitDetails.Repo, gitDetails.Branch, testGitPath+tsparams.ZtpKustomizationPath) {
+		return false, fmt.Errorf("git path '%s' could not be found", testGitPath)
+	}
+
+	gitDetails.Path = testGitPath
+	err := SetGitDetailsInArgoCd(appName, gitDetails, true, syncMustBeValid)
+
+	return true, err
+}
+
+// JoinGitPaths is used to join any combination of git strings but also avoiding double slashes.
+func JoinGitPaths(inputs []string) string {
+	// We want to preserve any existing double slashes but we don't want to add any between the input elements
+	// To work around this we will use a special join character and a couple replacements
+	special := "<<join>>"
+
+	// Combine the inputs with the special join character
+	result := strings.Join(
+		inputs,
+		special,
+	)
+
+	// Replace any special joins that have a slash prefix
+	result = strings.ReplaceAll(result, "/"+special, "/")
+
+	// Replace any special joins that have a slash suffix
+	result = strings.ReplaceAll(result, special+"/", "/")
+
+	// Finally replace any remaining special joins
+	result = strings.ReplaceAll(result, special, "/")
+
+	// The final result should never have double slashes between the joined elements
+	// However if they already had any double slashes, e.g. "http://", they will be preserved
+	return result
+}
+
+// DoesGitPathExist checks if the specified git url exists by sending an HTTP request to it.
+func DoesGitPathExist(gitURL, gitBranch, gitPath string) bool {
+	url := JoinGitPaths([]string{
+		strings.Replace(gitURL, ".git", "", 1),
+		"raw",
+		gitBranch,
+		gitPath,
+	})
+
+	glog.V(tsparams.LogLevel).Infof("Checking if git url '%s' exists", url)
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	resp, err := client.Get(url)
+	if err == nil && resp.StatusCode == 200 {
+		glog.V(tsparams.LogLevel).Infof("found valid git url for '%s'", gitPath)
+
+		return true
+	}
+
+	glog.V(tsparams.LogLevel).Infof("could not find valid url for '%s'", gitPath)
+
+	return false
+}
+
+// waitForArgoCdChangeToComplete waits up to the specified timeout for the Argo CD configuration to be updated.
+func waitForArgoCdChangeToComplete(appName string, syncMustBeValid bool, timeout time.Duration) error {
+	return wait.PollUntilContextTimeout(
+		context.TODO(), tsparams.ArgoCdChangeInterval, timeout, true, func(ctx context.Context) (done bool, err error) {
+			glog.V(tsparams.LogLevel).Infof("Checking if change to Argo CD app %s is complete", appName)
+
+			app, err := argocd.PullApplication(HubAPIClient, appName, ranparam.OpenshiftGitOpsNamespace)
+			if err != nil {
+				return false, err
+			}
+
+			for i, condition := range app.Object.Status.Conditions {
+				// If there are any conditions then it probably means theres a problem. By printing them
+				// here we can make diagnosing a failing test easier.
+				glog.V(tsparams.LogLevel).Infof("Argo CD app %s condition #%d: '%v'", appName, i, condition)
+			}
+
+			// The sync result may also have helpful information in the event of an error.
+			operationState := app.Object.Status.OperationState
+			if operationState != nil && operationState.SyncResult != nil {
+				for i, resource := range operationState.SyncResult.Resources {
+					if resource != nil {
+						glog.V(tsparams.LogLevel).Infof("Argo CD app %s sync resource #%d: '%v'", appName, i, resource)
+					}
+				}
+			}
+
+			statusSource := app.Object.Status.Sync.ComparedTo.Source
+			appSource := app.Object.Spec.Source
+
+			if statusSource.RepoURL == appSource.RepoURL &&
+				statusSource.TargetRevision == appSource.TargetRevision &&
+				statusSource.Path == appSource.Path {
+				if syncMustBeValid {
+					return app.Object.Status.Sync.Status == "Synced", nil
+				}
+
+				return true, nil
+			}
+
+			return false, nil
+		})
+}

tests/cnf/ran/ztp/internal/helper/helper.go

@@ -0,0 +1,56 @@
+package helper
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/openshift-kni/eco-goinfra/pkg/clients"
+	"github.com/openshift-kni/eco-goinfra/pkg/ocm"
+	"github.com/openshift-kni/eco-gotests/tests/cnf/ran/ztp/internal/tsparams"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+// WaitForPolicyToExist waits for up to the specified timeout until the policy exists.
+func WaitForPolicyToExist(
+	client *clients.Settings, name, namespace string, timeout time.Duration) (*ocm.PolicyBuilder, error) {
+	var policy *ocm.PolicyBuilder
+
+	err := wait.PollUntilContextTimeout(
+		context.TODO(), tsparams.ArgoCdChangeInterval, timeout, true, func(ctx context.Context) (bool, error) {
+			var err error
+			policy, err = ocm.PullPolicy(client, name, namespace)
+
+			if err == nil {
+				return true, nil
+			}
+
+			if strings.Contains(err.Error(), "does not exist") {
+				return false, nil
+			}
+
+			return false, err
+		})
+
+	return policy, err
+}
+
+// WaitUntilSearchCollectorEnabled waits up to timeout until the KAC has the search collector addon enabled.
+func WaitUntilSearchCollectorEnabled(kac *ocm.KACBuilder, timeout time.Duration) error {
+	glog.V(tsparams.LogLevel).Infof(
+		"Waiting until search collector is enabled for KAC %s in namespace %s", kac.Definition.Name, kac.Definition.Namespace)
+
+	return wait.PollUntilContextTimeout(
+		context.TODO(), tsparams.ArgoCdChangeInterval, timeout, true, func(ctx context.Context) (bool, error) {
+			if !kac.Exists() {
+				glog.V(tsparams.LogLevel).Infof(
+					"KAC %s in namespace %s does not exist", kac.Definition.Name, kac.Definition.Namespace)
+
+				return false, fmt.Errorf("kac %s in namespace %s does not exist", kac.Definition.Name, kac.Definition.Namespace)
+			}
+
+			return kac.Definition.Spec.SearchCollectorConfig.Enabled, nil
+		})
+}

tests/cnf/ran/ztp/internal/tsparams/consts.go

@@ -9,10 +9,19 @@ import (
 const (
 	// LabelSuite is the label for all the tests in this suite.
 	LabelSuite = "ran-ztp"
+	// LabelArgoCdAcmCrsTestCases is the label for the ACM CRs test cases.
+	LabelArgoCdAcmCrsTestCases = "ztp-argocd-acm-crs"
+	// LabelArgoCdClustersAppTestCases is the label for the Argo CD clusters app test cases.
+	LabelArgoCdClustersAppTestCases = "ztp-argocd-clusters"
 
 	// TestNamespace is the namespace used for ZTP tests.
 	TestNamespace = "ztp-test"
 
+	// MultiClusterHubOperator is the name of the multi cluster hub operator.
+	MultiClusterHubOperator = "multiclusterhub-operator"
+	// AcmPolicyGeneratorName is the name of the ACM policy generator container.
+	AcmPolicyGeneratorName = "acm-policy-generator"
+
 	// ArgoCdPoliciesAppName is the name of the policies app in Argo CD.
 	ArgoCdPoliciesAppName = "policies"
 	// ArgoCdClustersAppName is the name of the clusters app in Argo CD.
@@ -23,9 +32,18 @@ const (
 	// ArgoCdChangeTimeout is the time to use for polling for changes to Argo CD.
 	ArgoCdChangeTimeout = 10 * time.Minute
 
+	// ZtpTestPathAcmCrs is the git path for the ACM CRs test.
+	ZtpTestPathAcmCrs = "ztp-test/acm-crs"
+	// ZtpTestPathClustersApp is the git path for the clusters app test.
+	ZtpTestPathClustersApp = "ztp-test/klusterlet-addon"
+	// ZtpTestPathRemoveNmState is the git path for the remove nm state test.
+	ZtpTestPathRemoveNmState = "ztp-test/remove-nmstate"
 	// ZtpKustomizationPath is the path to the kustomization file in the ztp test.
 	ZtpKustomizationPath = "/kustomization.yaml"
 
+	// AcmCrsPolicyName is the name of the policy for ACM CRs.
+	AcmCrsPolicyName = "acm-crs-policy"
+
 	// LogLevel is the verbosity of glog statements in this test suite.
 	LogLevel glog.Level = 90
 )