cnf ran: fix post-provision automation issue

There are a few lingering automation issues caused by race conditions between the tests and the operator on the cluster. This PR addresses them in two ways:

1. Adds a sleep to 77374. This is a bit of a workaround but for now the simplest way to ensure that we do not end up in the situation where a policy template is updated but its compliance state refers to the old template.
1. Adds an additional step to waitForPolicies which ensures that the policy details in the ProvisioningRequest status match the desired policy version. Previously, we could end up in a situation where all the policies have propagated to the spoke but the ProvisioningRequest shows the old policies. This would then cause a 403 as the ProvisioningRequest turns pending after the policies have propagated and the state had been fulfilled.

A more permanent solution would likely be switching to watching resources instead of polling but the current solution requires the least changes for now.

tests/cnf/ran/oran/internal/helper/helper.go

@@ -227,3 +227,29 @@ func WaitForPolicyVersion(client *clients.Settings, namespace, policyVersion str
 			return true, nil
 		})
 }
+
+// WaitForPRPolicyVersion waits up to timeout until all of the policies on the provided ProvisioningRequest have
+// policyVersion.
+func WaitForPRPolicyVersion(
+	prBuilder *oran.ProvisioningRequestBuilder, policyVersion string, timeout time.Duration) error {
+	return wait.PollUntilContextTimeout(
+		context.TODO(), 3*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
+			// Exists will update the Object with the latest ProvisioningRequest.
+			if !prBuilder.Exists() {
+				glog.V(tsparams.LogLevel).Infof("Failed to verify ProvisioningRequest %s exists", prBuilder.Definition.Name)
+
+				return false, nil
+			}
+
+			for _, policyDetail := range prBuilder.Object.Status.Extensions.Policies {
+				if !strings.HasPrefix(policyDetail.PolicyName, policyVersion) {
+					glog.V(tsparams.LogLevel).Infof("Policy %s does not match expected policy version %s",
+						policyDetail.PolicyName, policyVersion)
+
+					return false, nil
+				}
+			}
+
+			return true, nil
+		})
+}

tests/cnf/ran/oran/tests/oran-post-provision.go

@@ -35,6 +35,10 @@ var _ = Describe("ORAN Post-provision Tests", Label(tsparams.LabelPostProvision)
 
 		copiedSpec := prBuilder.Definition.Spec
 		originalPRSpec = &copiedSpec
+
+		By("verifying ProvisioningRequest is fulfilled to start")
+		prBuilder, err = prBuilder.WaitUntilFulfilled(time.Minute)
+		Expect(err).ToNot(HaveOccurred(), "Failed to verify spoke 1 ProvisioningRequest is fulfilled")
 	})
 
 	AfterEach(func() {
@@ -109,6 +113,19 @@ var _ = Describe("ORAN Post-provision Tests", Label(tsparams.LabelPostProvision)
 		})
 
 		prBuilder = updatePRUntilNoConflict(prBuilder)
+
+		By("waiting to ensure the policy status updates")
+		// This test case updates a previously compliant policy so if we check the compliance state too soon we
+		// risk a situation where the policy has been updated but its compliance state has not been.
+		time.Sleep(15 * time.Second)
+
+		DeferCleanup(func() {
+			By("waiting to ensure the policy status updates")
+			// The same issue that happens on the cleanup side of this test case, so wait again to ensure
+			// the next test case is not affected.
+			time.Sleep(15 * time.Second)
+		})
+
 		waitForPolicies(prBuilder)
 
 		By("verifying the test ConfigMap has the new value")
@@ -193,18 +210,14 @@ var _ = Describe("ORAN Post-provision Tests", Label(tsparams.LabelPostProvision)
 
 	// 77379 - Failed update to ProvisioningRequest and successful rollback
 	It("successfully rolls back failed ProvisioningRequest update", reportxml.ID("77379"), func() {
-		By("verifying ProvisioningRequest is valid to start")
-		prBuilder, err := prBuilder.WaitForCondition(tsparams.PRConfigurationAppliedCondition, time.Minute)
-		Expect(err).ToNot(HaveOccurred(), "Failed to verify spoke 1 ProvisioningRequest has ConfigurationApplied")
-
 		By("updating the policyTemplateParameters")
 		prBuilder = prBuilder.WithTemplateParameter(tsparams.PolicyTemplateParamsKey, map[string]string{
 			tsparams.HugePagesSizeKey: "2G",
 		})
 		prBuilder = updatePRUntilNoConflict(prBuilder)
 
 		By("waiting for policy to go NonCompliant")
-		err = helper.WaitForNoncompliantImmutable(HubAPIClient, RANConfig.Spoke1Name, time.Minute)
+		err := helper.WaitForNoncompliantImmutable(HubAPIClient, RANConfig.Spoke1Name, time.Minute)
 		Expect(err).ToNot(HaveOccurred(), "Failed to wait for a spoke 1 policy to go NonCompliant due to immutable field")
 
 		By("fixing the policyTemplateParameters")
@@ -303,6 +316,12 @@ func waitForPolicies(prBuilder *oran.ProvisioningRequestBuilder) {
 	err = helper.WaitForPolicyVersion(Spoke1APIClient, RANConfig.Spoke1Name, policyVersion, 2*time.Minute)
 	Expect(err).ToNot(HaveOccurred(), "Failed to wait for policies to propagate to the spoke")
 
+	By("waiting for ProvisioningRequest to have the correct policies")
+
+	err = helper.WaitForPRPolicyVersion(prBuilder, policyVersion, 2*time.Minute)
+	Expect(err).ToNot(HaveOccurred(),
+		"Failed to wait for the ProvisioningRequest to have the correct policy version %s", policyVersion)
+
 	By("waiting for policies to be compliant")
 
 	err = ocm.WaitForAllPoliciesComplianceState(