update ResourceGroup CRD (#2794)

- Add the new fields for capturing the apply and reconcile status
- Update the pre-steps of apply. If the ResourceGroup CRD is
installed but doesn't match the latest declaration, the ResourceGroup
CRD should be re-applied.

e2e/live/end-to-end-test.sh

@@ -129,7 +129,30 @@ function downloadPreviousKpt {
   fi
   tar -xvf kpt.tar.gz > $OUTPUT_DIR/kptdownload 2>&1
   mv kpt $BIN_DIR/previouskpt
-  echo "Downloading latest kpt binary...SUCCESS"
+  echo "Downloading previous kpt binary...SUCCESS"
+  rm kpt.tar.gz LICENSES.txt lib.zip
+  set +e
+}
+
+function downloadKpt1.0 {
+  set -e
+  echo "Downloading v1.0.0-beta.13 kpt binary..."
+  uname="$(uname -s)"
+  if [[ "$uname" == "Linux" ]]
+  then
+    echo "Running on Linux"
+    curl -LJ -o kpt.tar.gz https://github.com/GoogleContainerTools/kpt/releases/download/v1.0.0-beta.13/kpt_linux_amd64-1.0.0-beta.13.tar.gz > $OUTPUT_DIR/kptdownload 2>&1
+  elif [[ "$uname" == "Darwin" ]]
+  then
+    echo "Running on Darwin"
+      curl -LJ -o kpt.tar.gz https://github.com/GoogleContainerTools/kpt/releases/download/v1.0.0-beta.13/kpt_darwin_amd64-1.0.0-beta.13.tar.gz > $OUTPUT_DIR/kptdownload 2>&1
+    else
+      echo "ERROR: Unknown OS $uname"
+      exit 1
+  fi
+  tar -xvf kpt.tar.gz > $OUTPUT_DIR/kptdownload 2>&1
+  mv kpt $BIN_DIR/kpt1.0.0
+  echo "Downloading 1.0.0 kpt binary...SUCCESS"
   rm kpt.tar.gz LICENSES.txt lib.zip
   set +e
 }
@@ -409,6 +432,7 @@ echo
 buildKpt
 
 downloadPreviousKpt
+downloadKpt1.0
 
 echo
 set +e                          # Do not stop the test for errors
@@ -836,6 +860,70 @@ assertPodExists "pod-c" "test-rg-namespace"
 assertPodExists "pod-d" "test-rg-namespace"
 printResult
 
+###########################################################
+#  Test Update ResourceGroup CRD during apply
+###########################################################
+
+createTestSuite
+waitForDefaultServiceAccount
+
+# This test first applies a kpt package with kpt1.0.0,
+# which uses the previous ResourceGroup CRD.
+# Then it re-apply the same kpt package with the built kpt,
+# which uses a new ResourceGroup CRD.
+# It updates the ResourceGroup CRD and apply/prune works as expected.
+echo "Testing apply with kpt 1.0.0 and re-apply/prune with built kpt"
+echo "cat e2e/live/testdata/stdin-test/pods.yaml | kpt1.0.0 live apply -"
+cat e2e/live/testdata/stdin-test/pods.yaml | ${BIN_DIR}/kpt1.0.0 live apply - > $OUTPUT_DIR/status 2>&1
+assertContains "pod/pod-a created"
+assertContains "pod/pod-b created"
+assertContains "pod/pod-c created"
+assertContains "4 resource(s) applied. 3 created, 1 unchanged, 0 configured, 0 failed"
+printResult
+echo "cat e2e/live/testdata/stdin-test/pods.yaml | kpt live apply -"
+cat e2e/live/testdata/stdin-test/pods.yaml | ${BIN_DIR}/kpt live apply - > $OUTPUT_DIR/status 2>&1
+assertContains "namespace/stdin-test-namespace unchanged"
+assertContains "pod/pod-a unchanged"
+assertContains "pod/pod-b unchanged"
+assertContains "pod/pod-c unchanged"
+printResult
+echo "cat e2e/live/testdata/stdin-test/pods.yaml | kpt live destroy -"
+cat e2e/live/testdata/stdin-test/pods.yaml | ${BIN_DIR}/kpt live destroy - > $OUTPUT_DIR/status 2>&1
+assertContains "pod/pod-a deleted"
+assertContains "pod/pod-b deleted"
+assertContains "pod/pod-c deleted"
+assertContains "4 resource(s) deleted, 0 skipped"
+printResult
+
+# Test: don't have permission to update the ResourceGroup CRD
+# should see an error message
+echo "Testing updating ResourceGroup CRD during apply"
+echo "Apply the previous ResourceGroup CRD"
+${BIN_DIR}/kpt1.0.0 live install-resource-group > $OUTPUT_DIR/status 2>&1
+echo "kubectl apply -f e2e/live/testdata/update-rg-crd"
+# Setup: create a service account and bind a Role to it so it has administrative
+# privileges on the "test" namespace, but no permissions to Get or Update CRD.
+kubectl apply -f e2e/live/testdata/rbac-error-step-1 > $OUTPUT_DIR/status
+assertContains "namespace/rbac-error created"
+assertContains "rolebinding.rbac.authorization.k8s.io/admin created"
+assertContains "serviceaccount/user created"
+wait 2
+
+# Setup: use the service account just created. It does not have permissions
+# on the default namespace, so it will give a permissions error on apply
+# for anything attempted to apply to the default namespace.
+kubectl config set-credentials user --token="$(kubectl get secrets -ojsonpath='{.data.token}' \
+  "$(kubectl get sa user -ojsonpath='{.secrets[0].name}')" \
+  | base64 -d)" > $OUTPUT_DIR/status
+kubectl config set-context kind-kind:user --cluster=kind-kind --user=user > $OUTPUT_DIR/status
+kubectl config use-context kind-kind:user > $OUTPUT_DIR/status
+wait 2
+
+# Attempt to apply a kpt package. It fails with an error message.
+${BIN_DIR}/kpt live apply e2e/live/testdata/rbac-error-step-2 > $OUTPUT_DIR/status 2>&1
+assertContains "error: Type ResourceGroup CRD needs update."
+printResult
+
 # Clean-up the k8s cluster
 echo "Cleaning up cluster"
 cp -f e2e/live/testdata/Kptfile e2e/live/testdata/rg-test-case-1a

internal/cmdapply/cmdapply.go

@@ -201,12 +201,20 @@ func runApply(r *Runner, invInfo inventory.InventoryInfo, objs []*unstructured.U
 	dryRunStrategy common.DryRunStrategy) error {
 	if r.installCRD {
 		f := r.factory
-		// Only install the ResourceGroup CRD if it is not already installed.
-		if err := cmdutil.VerifyResourceGroupCRD(f); err != nil {
-			err = cmdutil.InstallResourceGroupCRD(r.ctx, f)
-			if err != nil {
+		// Install the ResourceGroup CRD if it is not already installed
+		// or if the ResourceGroup CRD doesn't match the CRD in the
+		// kpt binary.
+		err := cmdutil.VerifyResourceGroupCRD(f)
+		if err != nil {
+			if err = cmdutil.InstallResourceGroupCRD(r.ctx, f); err != nil {
 				return err
 			}
+		} else if !live.ResourceGroupCRDMatched(f) {
+			if err = cmdutil.InstallResourceGroupCRD(r.ctx, f); err != nil {
+				return &cmdutil.ResourceGroupCRDNotLatestError{
+					Err: err,
+				}
+			}
 		}
 	}
 

internal/cmdutil/util.go

@@ -16,6 +16,7 @@ package cmdutil
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/GoogleContainerTools/kpt/internal/printer"
 	"github.com/GoogleContainerTools/kpt/pkg/live"
@@ -73,3 +74,16 @@ type NoResourceGroupCRDError struct{}
 func (*NoResourceGroupCRDError) Error() string {
 	return "type ResourceGroup not found"
 }
+
+// ResourceGroupCRDNotLatestError is an error type that will be used when a
+// cluster has a ResourceGroup CRD that doesn't match the
+// latest declaration.
+type ResourceGroupCRDNotLatestError struct{
+	Err error
+}
+
+func (e *ResourceGroupCRDNotLatestError) Error() string {
+	return fmt.Sprintf(
+		"Type ResourceGroup CRD needs update. Please make sure you have the permission " +
+			"to update CRD then run `kpt live install-resource-group`.\n %v",  e.Err)
+}

pkg/live/example-resource-group-crd-for-v1.yaml

@@ -149,6 +149,10 @@ spec:
                   description: ResourceStatus contains the status of a given resource
                     uniquely identified by its group, kind, name and namespace.
                   properties:
+                    actuation:
+                      description: actuation indicates whether actuation has been
+                        performed yet and how it went.
+                      type: string
                     conditions:
                       items:
                         properties:
@@ -184,9 +188,19 @@ spec:
                       type: string
                     namespace:
                       type: string
+                    reconcile:
+                      description: reconcile indicates whether reconciliation has
+                        been performed yet and how it went.
+                      type: string
+                    sourceHash:
+                      type: string
                     status:
                       description: Status describes the status of a resource
                       type: string
+                    strategy:
+                      description: strategy indicates the method of actuation (apply
+                        or delete) used or planned to be used.
+                      type: string
                   required:
                   - group
                   - kind

pkg/live/example-resource-group-crd.yaml

@@ -150,6 +150,10 @@ spec:
                 description: ResourceStatus contains the status of a given resource
                   uniquely identified by its group, kind, name and namespace.
                 properties:
+                  actuation:
+                    description: actuation indicates whether actuation has been
+                      performed yet and how it went.
+                    type: string
                   conditions:
                     items:
                       properties:
@@ -185,9 +189,19 @@ spec:
                     type: string
                   namespace:
                     type: string
+                  reconcile:
+                    description: reconcile indicates whether reconciliation has
+                      been performed yet and how it went.
+                    type: string
+                  sourceHash:
+                    type: string
                   status:
                     description: Status describes the status of a resource
                     type: string
+                  strategy:
+                    description: strategy indicates the method of actuation (apply
+                      or delete) used or planned to be used.
+                    type: string
                 required:
                 - group
                 - kind

pkg/live/inventoryrg.go

@@ -17,12 +17,14 @@ package live
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"strings"
 	"time"
 
 	"github.com/GoogleContainerTools/kpt/pkg/status"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/klog/v2"
@@ -244,6 +246,56 @@ func ResourceGroupCRDApplied(factory cmdutil.Factory) bool {
 	return true
 }
 
+// ResourceGroupCRDMatched checks if the ResourceGroup CRD
+// in the cluster matches the CRD in the kpt binary.
+func ResourceGroupCRDMatched(factory cmdutil.Factory) bool {
+	mapper, err := factory.ToRESTMapper()
+	if err != nil {
+		klog.V(4).Infof("error retrieving RESTMapper when checking ResourceGroup CRD: %s\n", err)
+		return false
+	}
+	crd, err := rgCRD(mapper)
+	if err != nil {
+		klog.V(7).Infof("failed to get ResourceGroup CRD from string: %s", err)
+		return false
+	}
+
+	dc, err := factory.DynamicClient()
+	if err != nil {
+		klog.V(7).Infof("error getting the dynamic client: %s\n", err)
+		return false
+	}
+
+	mapping, err := mapper.RESTMapping(crdGroupKind)
+	if err != nil {
+		klog.V(7).Infof("Failed to get mapping of CRD type: %s", err)
+		return false
+	}
+
+	liveCRD, err := dc.Resource(mapping.Resource).Get(context.TODO(), "resourcegroups.kpt.dev", metav1.GetOptions{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: crd.GetAPIVersion(),
+			Kind:       "CustomResourceDefinition",
+		},
+	})
+	if err != nil {
+		klog.V(7).Infof("error getting the ResourceGroup CRD from cluster: %s\n", err)
+		return false
+	}
+
+	liveSpec, _, err := unstructured.NestedMap(liveCRD.Object, "spec")
+	if err != nil {
+		klog.V(7).Infof("error getting the ResourceGroup CRD spec from cluster: %s\n", err)
+		return false
+	}
+	latestspec, _, err := unstructured.NestedMap(crd.Object, "spec")
+	if err != nil {
+		klog.V(7).Infof("error getting the ResourceGroup CRD spec from string: %s\n", err)
+		return false
+	}
+	return reflect.DeepEqual(liveSpec, latestspec)
+}
+
 // InstallResourceGroupCRD applies the custom resource definition for the
 // ResourceGroup by creating and running a TaskQueue of Tasks necessary.
 // The Tasks are 1) Apply CRD task, 2) Wait Task (for CRD to become
@@ -510,6 +562,10 @@ spec:
                 description: ResourceStatus contains the status of a given resource
                   uniquely identified by its group, kind, name and namespace.
                 properties:
+                  actuation:
+                    description: actuation indicates whether actuation has been
+                      performed yet and how it went.
+                    type: string
                   conditions:
                     items:
                       properties:
@@ -545,9 +601,19 @@ spec:
                     type: string
                   namespace:
                     type: string
+                  reconcile:
+                    description: reconcile indicates whether reconciliation has
+                      been performed yet and how it went.
+                    type: string
+                  sourceHash:
+                    type: string
                   status:
                     description: Status describes the status of a resource
                     type: string
+                  strategy:
+                    description: strategy indicates the method of actuation (apply
+                      or delete) used or planned to be used.
+                    type: string
                 required:
                 - group
                 - kind
@@ -716,6 +782,10 @@ spec:
                   description: ResourceStatus contains the status of a given resource
                     uniquely identified by its group, kind, name and namespace.
                   properties:
+                    actuation:
+                      description: actuation indicates whether actuation has been
+                        performed yet and how it went.
+                      type: string
                     conditions:
                       items:
                         properties:
@@ -751,9 +821,19 @@ spec:
                       type: string
                     namespace:
                       type: string
+                    reconcile:
+                      description: reconcile indicates whether reconciliation has
+                        been performed yet and how it went.
+                      type: string
+                    sourceHash:
+                      type: string
                     status:
                       description: Status describes the status of a resource
                       type: string
+                    strategy:
+                      description: strategy indicates the method of actuation (apply
+                        or delete) used or planned to be used.
+                      type: string
                   required:
                   - group
                   - kind