diff --git a/docs/user-resources.md b/docs/user-resources.md index f49ec59a7..d7520c0cf 100644 --- a/docs/user-resources.md +++ b/docs/user-resources.md @@ -15,37 +15,17 @@ Custom resources (not `CustomResourceDefinition`s) are not supported. The resources are applied in the following order according to their kind. - Namespace - - rank: 10 - ServiceAccount - - rank: 20 - CustomResourceDefinition - - rank: 30 - ClusterRole - - rank: 40 - ClusterRoleBinding - - rank: 50 - (Other cluster-scope resources) - - rank: 1000 - Role - - rank: 2000 - RoleBinding - - rank: 2010 - NetworkPolicy - - rank: 2020 - Secret - - rank: 2030 - ConfigMap - - rank: 2040 - (Other namespace-scoped resources) - - rank: 3000 - -### Custom order - -Users can customize the order of applying resources by annotating `cke.cybozu.com/rank`. -In the case of cluster-scoped resources, a rank value must be 0-1999. -For namespace-scoped resources, a rank value must be 2000-. - -If `cke.cybozu.com/rank` is not set, the rank is defaulted to a value based on the aforementioned list. ## Annotations diff --git a/mtest/kubernetes_test.go b/mtest/kubernetes_test.go index 4d2bdfb84..0d88071f0 100644 --- a/mtest/kubernetes_test.go +++ b/mtest/kubernetes_test.go @@ -399,7 +399,7 @@ rules: clusterSetAndWait(cluster) }) - It("should create user-defined resources", func() { + It("updates user-defined resources", func() { By("set user-defined resource") resources := `apiVersion: v1 kind: Namespace @@ -435,115 +435,11 @@ roleRef: kind: Role name: pod-reader apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: foo - name: test-deployment - annotations: - cke.cybozu.com/rank: "2200" -spec: - replicas: 1 - selector: - matchLabels: - cke.cybozu.com/appname: test-deployment - template: - metadata: - labels: - cke.cybozu.com/appname: test-deployment - spec: - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: test-daemonset - namespace: foo - annotations: - cke.cybozu.com/rank: "2100" -spec: - selector: - matchLabels: - cke.cybozu.com/appname: test-daemonset - template: - metadata: - labels: - cke.cybozu.com/appname: test-daemonset - spec: - initContainers: - - name: wait-10sec - image: quay.io/cybozu/ubuntu:22.04 - args: - - sleep - - "10" - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause ` - - By("checking the order of applying") - _, _, err := ckecliWithInput([]byte(resources), "resource", "set", "-") - Expect(err).NotTo(HaveOccurred()) - + ckecliWithInput([]byte(resources), "resource", "set", "-") defer ckecliWithInput([]byte(resources), "resource", "delete", "-") ts := time.Now() - By("waiting to complete creation") - Eventually(func() error { - stdout, _, err := kubectl("-n", "foo", "get", "daemonset", "test-daemonset", "-o", "json") - if err != nil { - return err - } - var ds appsv1.DaemonSet - if err := json.Unmarshal(stdout, &ds); err != nil { - return err - } - desired := ds.Status.DesiredNumberScheduled - if desired == 0 { - return fmt.Errorf("desired must be not 0") - } - available := ds.Status.NumberAvailable - updated := ds.Status.UpdatedNumberScheduled - stdout, _, err = kubectl("-n", "foo", "get", "pods", "-l", "cke.cybozu.com/appname=test-deployment", "-o", "json") - if err != nil { - return err - } - podList := &corev1.PodList{} - if err := json.Unmarshal(stdout, podList); err != nil { - return err - } - - if (desired != available) || (desired != updated) { - // We expect that the array of pod.Items is empty - if len(podList.Items) == 0 { - return fmt.Errorf("should wait") - } - // If the array is not empty, test must fail and return - Fail("Deployment(foo/test-deployment) resource is expected not to create before completing to create DaemonSet(test-daemonset)") - } - return nil - }).WithTimeout(2 * time.Minute).Should(Succeed()) - - By("getting deployment") - Eventually(func() error { - dp := &appsv1.Deployment{} - stdout, _, err := kubectl("-n", "foo", "get", "deployment", "test-deployment", "-o", "json") - if err != nil { - return err - } - if err := json.Unmarshal(stdout, dp); err != nil { - return err - } - return nil - }).Should(Succeed()) - cluster := getCluster() for i := 0; i < 3; i++ { cluster.Nodes[i].ControlPlane = true @@ -552,487 +448,27 @@ spec: return checkCluster(cluster, ts) }).Should(Succeed()) - }) - - It("should update user-defined resources", func() { - resources := `apiVersion: v1 + By("updating user-defined resources") + newResources := `apiVersion: v1 kind: Namespace metadata: name: foo labels: test: value ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: foo - name: test-deployment - annotations: - cke.cybozu.com/rank: "2200" - labels: - updated: "true" -spec: - replicas: 1 - selector: - matchLabels: - cke.cybozu.com/appname: test-deployment - template: - metadata: - labels: - cke.cybozu.com/appname: test-deployment - updated: "true" - spec: - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: test-daemonset - namespace: foo - annotations: - cke.cybozu.com/rank: "2100" - labels: - updated: "true" -spec: - selector: - matchLabels: - cke.cybozu.com/appname: test-daemonset - template: - metadata: - labels: - cke.cybozu.com/appname: test-daemonset - updated: "true" - spec: - initContainers: - - name: wait-10sec - image: quay.io/cybozu/ubuntu:22.04 - args: - - sleep - - "10" - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause ` - ckecliWithInput([]byte(resources), "resource", "set", "-") - defer ckecliWithInput([]byte(resources), "resource", "delete", "-") - ts := time.Now() - - Eventually(func() error { - stdout, _, err := kubectl("get", "namespaces/foo", "-o", "json") - if err != nil { - return err - } - var ns corev1.Namespace - if err := json.Unmarshal(stdout, &ns); err != nil { - return err - } - val, ok := ns.Labels["test"] - if !ok || val != "value" { - return fmt.Errorf("ns must have the label that key is test and value is value") - } - return nil - }).Should(Succeed()) - - By("waiting to complete the update of DaemonSet") - Eventually(func() error { - stdout, _, err := kubectl("-n", "foo", "get", "daemonset", "-l", "updated=true", "-o", "json") - if err != nil { - return err - } - var dsList appsv1.DaemonSetList - if err := json.Unmarshal(stdout, &dsList); err != nil { - return err - } - if len(dsList.Items) == 0 { - return fmt.Errorf("should wait to update") - } - // if the label is not exist, retry - desired := dsList.Items[0].Status.DesiredNumberScheduled - if desired == 0 { - return fmt.Errorf("desired must be not 0") - } - available := dsList.Items[0].Status.NumberAvailable - updated := dsList.Items[0].Status.UpdatedNumberScheduled - - stdout, _, err = kubectl("-n", "foo", "get", "deployment", "-l", "updated=true", "-o", "json") - if err != nil { - return err - } - dpList := appsv1.DeploymentList{} - if err := json.Unmarshal(stdout, &dpList); err != nil { - return err - } - - if (desired != available) || (desired != updated) { - // We expect that the array of DeploymentList.Items is empty - if len(dpList.Items) == 0 { - return fmt.Errorf("should wait") - } - // If the array is not empty, test must fail and return - Fail("Deployment(foo/test-deployment) resource is expected not to create before completing to create DaemonSet(test-daemonset)") - } - return nil - }).Should(Succeed()) - - By("getting deployment") - Eventually(func() error { - dp := &appsv1.Deployment{} - stdout, _, err := kubectl("-n", "foo", "get", "deployment", "test-deployment", "-o", "json") - if err != nil { - return err - } - if err := json.Unmarshal(stdout, dp); err != nil { - return err - } - l, ok := dp.Labels["updated"] - if !ok || l != "true" { - return fmt.Errorf("test-deployment must has the label named update") - } - return nil - }).Should(Succeed()) - - cluster := getCluster() - for i := 0; i < 3; i++ { - cluster.Nodes[i].ControlPlane = true - } - Eventually(func() error { - return checkCluster(cluster, ts) - }).Should(Succeed()) - - }) - - It("should change the order of user-defined resources", func() { - - resources := `apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: foo - name: test-deployment - annotations: - cke.cybozu.com/rank: "2050" - labels: - updated: "true" - changed: "true" -spec: - replicas: 1 - selector: - matchLabels: - cke.cybozu.com/appname: test-deployment - template: - metadata: - labels: - cke.cybozu.com/appname: test-deployment - updated: "true" - changed: "true" - spec: - initContainers: - - name: wait-30sec - image: quay.io/cybozu/ubuntu:22.04 - args: - - sleep - - "30" - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: test-daemonset - namespace: foo - annotations: - cke.cybozu.com/rank: "2100" - labels: - updated: "true" - changed: "true" -spec: - selector: - matchLabels: - cke.cybozu.com/appname: test-daemonset - template: - metadata: - labels: - cke.cybozu.com/appname: test-daemonset - updated: "true" - changed: "true" - spec: - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause -` - - ckecliWithInput([]byte(resources), "resource", "set", "-") - defer ckecliWithInput([]byte(resources), "resource", "delete", "-") - ts := time.Now() - - By("waiting to complete the update of Deployment") - Eventually(func() error { - stdout, _, err := kubectl("-n", "foo", "get", "deployment", "-l", "changed=true", "-o", "json") - if err != nil { - return err - } - var dpList appsv1.DeploymentList - if err := json.Unmarshal(stdout, &dpList); err != nil { - return err - } - if len(dpList.Items) == 0 { - return fmt.Errorf("should wait to update") - } - // if the label is not exist, retry - desired := dpList.Items[0].Status.Replicas - if desired == 0 { - return fmt.Errorf("desired must be not 0") - } - available := dpList.Items[0].Status.AvailableReplicas - updated := dpList.Items[0].Status.UpdatedReplicas - - stdout, _, err = kubectl("-n", "foo", "get", "daemonset", "-l", "changed=true", "-o", "json") - if err != nil { - return err - } - var dsList appsv1.DaemonSetList - if err := json.Unmarshal(stdout, &dsList); err != nil { - return err - } - - if (desired != available) || (desired != updated) { - // We expect that the array of DaemonsSetList.Items is empty - if len(dsList.Items) == 0 { - return fmt.Errorf("should wait") - } - // If the array is not empty, test must fail and return - Fail("DaemonSet(foo/test-daemonset) resource is expected not to create before completing to create Deployment(test-deployment)") - } - return nil - }).Should(Succeed()) - - By("getting daemonset") - Eventually(func() error { - ds := &appsv1.DaemonSet{} - stdout, _, err := kubectl("-n", "foo", "get", "daemonset", "test-daemonset", "-o", "json") - if err != nil { - return err - } - if err := json.Unmarshal(stdout, ds); err != nil { - return err - } - l, ok := ds.Labels["changed"] - if !ok || l != "true" { - return fmt.Errorf("test-daemonset must has the label named update") - } - return nil - }).Should(Succeed()) - - cluster := getCluster() - for i := 0; i < 3; i++ { - cluster.Nodes[i].ControlPlane = true - } - Eventually(func() error { - return checkCluster(cluster, ts) - }).Should(Succeed()) - }) - - It("should create the resource with default rank and update it with the custom rank", func() { - resources := `apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: foo - name: test-deployment2 - labels: - cke.cybozu.com/appname: test-deployment2 -spec: - replicas: 1 - selector: - matchLabels: - cke.cybozu.com/appname: test-deployment2 - template: - metadata: - labels: - cke.cybozu.com/appname: test-deployment2 - spec: - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: foo - name: test-deployment3 - labels: - cke.cybozu.com/appname: test-deployment3 -spec: - replicas: 1 - selector: - matchLabels: - cke.cybozu.com/appname: test-deployment3 - template: - metadata: - labels: - cke.cybozu.com/appname: test-deployment3 - spec: - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause -` - - By("creating new resources with default rank") - ckecliWithInput([]byte(resources), "resource", "set", "-") - defer ckecliWithInput([]byte(resources), "resource", "delete", "-") - ts := time.Now() - - cluster := getCluster() - for i := 0; i < 3; i++ { - cluster.Nodes[i].ControlPlane = true - } - Eventually(func() error { - return checkCluster(cluster, ts) - }).Should(Succeed()) - - newResources := `apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: foo - name: test-deployment2 - annotations: - cke.cybozu.com/rank: "2400" - labels: - cke.cybozu.com/appname: test-deployment2 - updated: "true" -spec: - replicas: 1 - selector: - matchLabels: - cke.cybozu.com/appname: test-deployment2 - template: - metadata: - labels: - cke.cybozu.com/appname: test-deployment2 - spec: - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: foo - name: test-deployment3 - annotations: - cke.cybozu.com/rank: "2300" - labels: - cke.cybozu.com/appname: test-deployment3 - updated: "true" -spec: - replicas: 1 - selector: - matchLabels: - cke.cybozu.com/appname: test-deployment3 - template: - metadata: - labels: - cke.cybozu.com/appname: test-deployment3 - spec: - initContainers: - - name: wait-30sec - image: quay.io/cybozu/ubuntu:22.04 - args: - - sleep - - "30" - containers: - - name: ubuntu - image: quay.io/cybozu/ubuntu:22.04 - args: - - pause -` - - By("updating resources with custom ranks") ckecliWithInput([]byte(newResources), "resource", "set", "-") defer ckecliWithInput([]byte(newResources), "resource", "delete", "-") ts = time.Now() - - By("waiting to complete the update of test-deployment3") - Eventually(func() error { - stdout, _, err := kubectl("-n", "foo", "get", "deployment", "-l", "updated=true,cke.cybozu.com/appname=test-deployment3", "-o", "json") - if err != nil { - return err - } - var dpList appsv1.DeploymentList - if err := json.Unmarshal(stdout, &dpList); err != nil { - return err - } - if len(dpList.Items) == 0 { - return fmt.Errorf("should wait to update") - } - // if the label is not exist, retry - desired := dpList.Items[0].Status.Replicas - if desired == 0 { - return fmt.Errorf("desired must be not 0") - } - available := dpList.Items[0].Status.AvailableReplicas - updated := dpList.Items[0].Status.UpdatedReplicas - - stdout, _, err = kubectl("-n", "foo", "get", "deployment", "-l", "updated=true,cke.cybozu.com/appname=test-deployment2", "-o", "json") - if err != nil { - return err - } - var dpList2 appsv1.DaemonSetList - if err := json.Unmarshal(stdout, &dpList2); err != nil { - return err - } - - if (desired != available) || (desired != updated) { - // We expect that the array of DaemonsSetList.Items is empty - if len(dpList2.Items) == 0 { - return fmt.Errorf("should wait") - } - // If the array is not empty, test must fail and return - Fail("foo/test-daemonset2 resource is expected not to create before completing to create foo/test-deployment3") - } - return nil - }).Should(Succeed()) - - By("getting test-deployment2") - Eventually(func() error { - dp := &appsv1.Deployment{} - stdout, _, err := kubectl("-n", "foo", "get", "deployment", "test-deployment2", "-o", "json") - if err != nil { - return err - } - if err := json.Unmarshal(stdout, dp); err != nil { - return err - } - l, ok := dp.Labels["updated"] - if !ok || l != "true" { - return fmt.Errorf("test-deployment2 must has the label named update") - } - return nil - }).Should(Succeed()) - - cluster = getCluster() - for i := 0; i < 3; i++ { - cluster.Nodes[i].ControlPlane = true - } Eventually(func() error { return checkCluster(cluster, ts) }).Should(Succeed()) + stdout, _, err := kubectl("get", "namespaces/foo", "-o", "json") + Expect(err).ShouldNot(HaveOccurred()) + var ns corev1.Namespace + err = json.Unmarshal(stdout, &ns) + Expect(err).ShouldNot(HaveOccurred()) + Expect(ns.Labels).Should(HaveKeyWithValue("test", "value")) }) It("embed certificates for webhooks", func() { diff --git a/op/nop.go b/op/nop.go deleted file mode 100644 index 3f6ce9be3..000000000 --- a/op/nop.go +++ /dev/null @@ -1,37 +0,0 @@ -package op - -import ( - "context" - - "github.com/cybozu-go/cke" -) - -type nopOp struct{} - -func NopOp() cke.Operator { - return &nopOp{} -} - -func (o *nopOp) Name() string { - return "nop" -} - -func (o *nopOp) NextCommand() cke.Commander { - return nil -} - -func (o *nopOp) Targets() []string { - return nil -} - -func (o *nopOp) Run(ctx context.Context, inf cke.Infrastructure, _ string) error { - // This function is never executed - return nil -} - -func (o *nopOp) Command() cke.Command { - return cke.Command{ - Name: "nop", - Target: "", - } -} diff --git a/op/status.go b/op/status.go index bd4fca66f..81d27f89e 100644 --- a/op/status.go +++ b/op/status.go @@ -436,71 +436,12 @@ func GetKubernetesClusterStatus(ctx context.Context, inf cke.Infrastructure, n * if err != nil { return cke.KubernetesClusterStatus{}, err } - if obj.GroupVersionKind().Kind == "DaemonSet" { - generation, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "metadata", "generation") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - observedGeneration, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "observedGeneration") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - - desired, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "desiredNumberScheduled") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - updated, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "updatedNumberScheduled") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - available, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "numberAvailable") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - s.SetResourceStatus(res.Key, obj.GetAnnotations(), len(obj.GetManagedFields()) != 0, objStatus(generation, observedGeneration, desired, updated, available)) - } else if obj.GroupVersionKind().Kind == "Deployment" { - generation, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "metadata", "generation") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - observedGeneration, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "observedGeneration") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - desired, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "readyReplicas") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - updated, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "updatedReplicas") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - available, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "replicas") - if err != nil { - return cke.KubernetesClusterStatus{}, err - } - s.SetResourceStatus(res.Key, obj.GetAnnotations(), len(obj.GetManagedFields()) != 0, objStatus(generation, observedGeneration, desired, updated, available)) - } else { - s.SetResourceStatus(res.Key, obj.GetAnnotations(), len(obj.GetManagedFields()) != 0, true) - } + s.SetResourceStatus(res.Key, obj.GetAnnotations(), len(obj.GetManagedFields()) != 0) } return s, nil } -func objStatus(generation, observedGeneration, desired, updated, available int64) bool { - if generation > observedGeneration { - return false - } - // If we get the status immediately after applying the resource, the value of desired may be 0. - // In this case, we need to return false. - if desired == 0 { - return false - } - return (desired == available) && (desired == updated) -} - func getClusterDNSStatus(ctx context.Context, inf cke.Infrastructure, n *cke.Node) (cke.ClusterDNSStatus, error) { clientset, err := inf.K8sClient(ctx, n) if err != nil { diff --git a/pkg/compile_resources/main.go b/pkg/compile_resources/main.go index 45bb09852..88e2da33c 100644 --- a/pkg/compile_resources/main.go +++ b/pkg/compile_resources/main.go @@ -94,7 +94,6 @@ func loadResources(fname string, images map[string]string) ([]cke.ResourceDefini Annotations struct { Revision int64 `json:"cke.cybozu.com/revision,string"` Image string `json:"cke.cybozu.com/image"` - Rank uint32 `json:"cke.cybozu.com/rank,omitempty,string"` } `json:"annotations"` } `json:"metadata"` }{} @@ -106,11 +105,6 @@ func loadResources(fname string, images map[string]string) ([]cke.ResourceDefini return nil, errors.New("static resources must declare cke.cybozu.com/revision in annotations") } - rank, err := cke.DecideRank(kind, namespace, obj.Metadata.Annotations.Rank) - if err != nil { - return nil, err - } - res = append(res, cke.ResourceDefinition{ Key: key, Kind: kind, @@ -118,7 +112,6 @@ func loadResources(fname string, images map[string]string) ([]cke.ResourceDefini Name: name, Revision: rev, Image: obj.Metadata.Annotations.Image, - Rank: rank, Definition: data, }) } @@ -143,7 +136,6 @@ var Resources = []cke.ResourceDefinition{ Name: {{ printf "%q" .Name }}, Revision: {{ .Revision }}, Image: {{ printf "%q" .Image }}, - Rank: {{ .Rank }}, Definition: []byte({{ printf "%q" .Definition }}), }, {{ end -}} diff --git a/resource.go b/resource.go index d9a0197de..04f4f159d 100644 --- a/resource.go +++ b/resource.go @@ -28,7 +28,6 @@ const ( AnnotationResourceRevision = "cke.cybozu.com/revision" AnnotationResourceInjectCA = "cke.cybozu.com/inject-cacert" AnnotationResourceIssueCert = "cke.cybozu.com/issue-cert" - AnnotationResourceRank = "cke.cybozu.com/rank" ) // kinds @@ -39,22 +38,6 @@ const ( KindValidatingWebhookConfiguration = "ValidatingWebhookConfiguration" ) -// rank -const ( - RankNamespace = 10 - RankServiceAccount = 20 // ServiceAccount is namespace scoped - RankCustomResourceDefinition = 30 - RankClusterRole = 40 - RankClusterRoleBinding = 50 - RankClusterScopedResourceDefault = 1000 - RankRole = 2000 - RankRoleBinding = 2010 - RankNetworkPolicy = 2020 - RankSecret = 2030 - RankConfigMap = 2040 - RankNamespaceScopedResourceDefault = 3000 -) - var decUnstructured = yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) // ApplyResource creates or updates given resource using server-side-apply. @@ -242,48 +225,6 @@ func ParseResource(data []byte) (string, error) { return obj.GetKind() + "/" + obj.GetNamespace() + "/" + name, nil } -func DecideRank(kind, namespace string, rank uint32) (uint32, error) { - if rank > 0 { - if namespace == "" && rank < 2000 { - return rank, nil - } - if namespace != "" && rank > 1999 { - return rank, nil - } - return 0, errors.New("invalid rank value") - } - // return default rank - switch kind { - case "Namespace": - return RankNamespace, nil - case "ServiceAccount": - return RankServiceAccount, nil - case "CustomResourceDefinition": - return RankCustomResourceDefinition, nil - case "ClusterRole": - return RankClusterRole, nil - case "ClusterRoleBinding": - return RankClusterRoleBinding, nil - // other cluster-scoped resources: 1000 - case "Role": - return RankRole, nil - case "RoleBinding": - return RankRoleBinding, nil - case "NetworkPolicy": - return RankNetworkPolicy, nil - case "Secret": - return RankSecret, nil - case "ConfigMap": - return RankConfigMap, nil - // other namespace scoped resources: 3000 - } - - if namespace == "" { - return RankClusterScopedResourceDefault, nil - } - return RankNamespaceScopedResourceDefault, nil -} - // ResourceDefinition represents a CKE-managed kubernetes resource. type ResourceDefinition struct { Key string @@ -292,7 +233,6 @@ type ResourceDefinition struct { Name string Revision int64 Image string // may contains multiple images; we should not use this whole string as an image name. - Rank uint32 Definition []byte } @@ -326,13 +266,45 @@ func (d ResourceDefinition) NeedUpdate(rs *ResourceStatus) bool { return curImage != d.Image } +func (d ResourceDefinition) rank() int { + switch d.Kind { + case "Namespace": + return 10 + case "ServiceAccount": + return 20 + case "CustomResourceDefinition": + return 30 + case "ClusterRole": + return 40 + case "ClusterRoleBinding": + return 50 + // other cluster-scoped resources: 1000 + case "Role": + return 2000 + case "RoleBinding": + return 2010 + case "NetworkPolicy": + return 2020 + case "Secret": + return 2030 + case "ConfigMap": + return 2040 + // other namespace scoped resources: 3000 + } + + if d.Namespace == "" { + return 1000 + } + return 3000 +} + // SortResources sort resources as defined order of creation. func SortResources(res []ResourceDefinition) { less := func(i, j int) bool { a := res[i] b := res[j] - aRank := a.Rank - bRank := b.Rank + aRank := a.rank() + bRank := b.rank() if aRank == bRank { return a.Key < b.Key diff --git a/server/control.go b/server/control.go index 07ffc9fb7..287100510 100644 --- a/server/control.go +++ b/server/control.go @@ -312,6 +312,7 @@ func (c Controller) runOnce(ctx context.Context, leaderKey string, tick <-chan t if err != nil { return err } + rqEntries, err := inf.Storage().GetRebootsEntries(ctx) if err != nil { return err @@ -390,12 +391,6 @@ func (c Controller) runOnce(ctx context.Context, leaderKey string, tick <-chan t } func runOp(ctx context.Context, op cke.Operator, leaderKey string, storage cke.Storage, inf cke.Infrastructure) error { - // Not to register the nopOp log in the history - if op.Name() == "nop" { - log.Info("execute nop op", map[string]interface{}{}) - return nil - } - // register operation record id, err := storage.NextRecordID(ctx) if err != nil { diff --git a/server/strategy.go b/server/strategy.go index 6e660cfbb..47e9820ff 100644 --- a/server/strategy.go +++ b/server/strategy.go @@ -612,17 +612,14 @@ func decideEtcdServiceOps(apiServer *cke.Node, svc *corev1.Service) cke.Operator } func decideResourceOps(apiServer *cke.Node, ks cke.KubernetesClusterStatus, resources []cke.ResourceDefinition, isReady bool) (ops []cke.Operator) { - for _, res := range static.Resources { // To avoid thundering herd problem. Deployments need to be created only after enough nodes become ready. if res.Kind == cke.KindDeployment && !isReady { continue } - // We don't have to wait to complete resource creation or update, status, ok := ks.ResourceStatuses[res.Key] if !ok || res.NeedUpdate(&status) { ops = append(ops, op.ResourceApplyOp(apiServer, res, !status.HasBeenSSA)) - return ops } } for _, res := range resources { @@ -630,28 +627,8 @@ func decideResourceOps(apiServer *cke.Node, ks cke.KubernetesClusterStatus, reso continue } status, ok := ks.ResourceStatuses[res.Key] - if !ok { + if !ok || res.NeedUpdate(&status) { ops = append(ops, op.ResourceApplyOp(apiServer, res, !status.HasBeenSSA)) - // To wait for the completion to create or update, we avoid applying subsequent resources. - return ops - } else { - if res.NeedUpdate(&status) { - ops = append(ops, op.ResourceApplyOp(apiServer, res, !status.HasBeenSSA)) - // To avoid applying subsequent resources not to wait the completion the update or creation - return ops - } else { - if !status.Completed { - log.Info("need to wait", map[string]interface{}{ - "resource_name": res.Name, - "resource_namespace": res.Namespace, - "kind": res.Kind, - "completed": status.Completed, - }) - ops = append(ops, op.NopOp()) - // To avoid applying subsequent resources not to wait the completion the update or creation - return ops - } - } } } return ops diff --git a/server/strategy_test.go b/server/strategy_test.go index c71727717..97aa8cc42 100644 --- a/server/strategy_test.go +++ b/server/strategy_test.go @@ -1,7 +1,6 @@ package server import ( - "strconv" "testing" "time" @@ -166,10 +165,7 @@ func newData() testData { NodeStatuses: nodeStatuses, Kubernetes: cke.KubernetesClusterStatus{ ResourceStatuses: map[string]cke.ResourceStatus{ - "Namespace/foo": { - Annotations: map[string]string{cke.AnnotationResourceRevision: "1"}, - Completed: true, - }, + "Namespace/foo": {Annotations: map[string]string{cke.AnnotationResourceRevision: "1"}}, }, Nodes: nodeList, }, @@ -193,30 +189,6 @@ func (d testData) withResources(res []cke.ResourceDefinition) testData { return d } -func (d testData) withResourcesReady(res []cke.ResourceDefinition) testData { - ks := &d.Status.Kubernetes - - d.Resources = append(d.Resources, res...) - - for _, r := range res { - status := cke.ResourceStatus{ - Annotations: map[string]string{cke.AnnotationResourceRevision: strconv.Itoa(int(r.Revision))}, - } - if r.Image != "" { - status.Annotations[cke.AnnotationResourceImage] = r.Image - } - ks.ResourceStatuses[r.Key] = status - } - return d -} - -func (d testData) withResourceStatus(statuses map[string]cke.ResourceStatus) testData { - for k, v := range statuses { - d.Status.Kubernetes.ResourceStatuses[k] = v - } - return d -} - func (d testData) withRivers() testData { for _, v := range d.Status.NodeStatuses { v.Rivers.Running = true @@ -413,7 +385,6 @@ func (d testData) withK8sResourceReady() testData { for _, res := range static.Resources { ks.ResourceStatuses[res.Key] = cke.ResourceStatus{ Annotations: map[string]string{cke.AnnotationResourceRevision: "1"}, - Completed: true, } } ks.ResourceStatuses["ClusterRole/system:cluster-dns"].Annotations[cke.AnnotationResourceRevision] = "2" @@ -1024,6 +995,14 @@ func TestDecideOps(t *testing.T) { Name: "K8sResources", Input: newData().withK8sReady(), ExpectedOps: []opData{ + {"resource-apply", 1}, + {"resource-apply", 1}, + {"resource-apply", 1}, + {"resource-apply", 1}, + {"resource-apply", 1}, + {"resource-apply", 1}, + {"resource-apply", 1}, + {"resource-apply", 1}, {"resource-apply", 1}, {"create-cluster-dns-configmap", 1}, {"create-endpoints", 1}, @@ -1250,161 +1229,30 @@ func TestDecideOps(t *testing.T) { { Name: "UserResourceAdd", Input: newData().withK8sResourceReady().withResources( - append(testResources, []cke.ResourceDefinition{ - { - Key: "ConfigMap/foo/bar", - Kind: "ConfigMap", - Namespace: "foo", - Name: "bar", - Revision: 1, - Definition: []byte(`{"apiversion":"v1","kind":"ConfigMap","metadata":{"namespace":"foo","name":"bar"},"data":{"a":"b"}}`), - }, - { - Key: "DaemonSet/foo/test-daemonset", - Kind: "DaemonSet", - Namespace: "foo", - Name: "test-daemonset", - Image: "test", - Revision: 1, - Definition: []byte(`{"apiVersion":"v1","kind":"DaemonSet","metadata":{"name":"test-daemonset", "namespace": "test"}}`), - }, - { - Key: "Deployment/foo/test-deployment", - Kind: "Deployment", - Namespace: "foo", - Name: "test-deployment", - Image: "test", - Revision: 1, - Definition: []byte(`{"apiVersion":"v1","kind":"Deployment","metadata":{"name":"test-deployment", "namespace": "test"}}`), - }, - }...)).withResourceStatus( - map[string]cke.ResourceStatus{ - "ConfigMap/foo/bar": { - Completed: true, - }, - "DaemonSet/foo/test-daemonset": { - Annotations: map[string]string{ - cke.AnnotationResourceRevision: "1", - cke.AnnotationResourceImage: "test", - }, - Completed: false, - }, - }, - ), - ExpectedOps: []opData{ - {"resource-apply", 1}, - }, - }, - { - Name: "UserResourceUpdate", - Input: newData().withK8sResourceReady().withResourcesReady([]cke.ResourceDefinition{ - { - Key: "DaemonSet/foo/test-daemonset", - Kind: "DaemonSet", - Namespace: "foo", - Name: "test-daemonset", - Image: "test", - Revision: 2, - Definition: []byte(`{"apiVersion":"v1","kind":"DaemonSet","metadata":{"name":"test-daemonset", "namespace": "test"}}`), - }, - { - Key: "Deployment/foo/test-deployment", - Kind: "Deployment", + append(testResources, cke.ResourceDefinition{ + Key: "ConfigMap/foo/bar", + Kind: "ConfigMap", Namespace: "foo", - Name: "test-deployment", - Image: "test", + Name: "bar", Revision: 1, - Definition: []byte(`{"apiVersion":"v1","kind":"Deployment","metadata":{"name":"test-deployment", "namespace": "test"}}`), - }, - }).withResources( - []cke.ResourceDefinition{ - { - Key: "Namespace/foo", - Kind: "Namespace", - Name: "foo", - Revision: 2, - Definition: []byte(`{"apiversion":"v1","kind":"Namespace","metadata":{"name":"foo"}}`), - }, - { - Key: "DaemonSet/foo/test-daemonset", - Kind: "DaemonSet", - Namespace: "foo", - Name: "test-daemonset", - Image: "test", - Revision: 2, - Definition: []byte(`{"apiVersion":"v1","kind":"DaemonSet","metadata":{"name":"test-daemonset", "namespace": "test"}}`), - }, - { - Key: "Deployment/foo/test-deployment", - Kind: "Deployment", - Namespace: "foo", - Name: "test-deployment", - Image: "test", - Revision: 2, - Definition: []byte(`{"apiVersion":"v1","kind":"Deployment","metadata":{"name":"test-deployment", "namespace": "test"}}`), - }, - }).withResourceStatus(map[string]cke.ResourceStatus{ - "Namespace/foo": { - Completed: true, - }, - "DaemonSet/foo/test-daemonset": { - Annotations: map[string]string{ - cke.AnnotationResourceRevision: "2", - cke.AnnotationResourceImage: "test", - }, - Completed: false, - }, - "Deployment/foo/test-deployment": { - Annotations: map[string]string{ - cke.AnnotationResourceRevision: "1", - cke.AnnotationResourceImage: "test", - }, - Completed: false, - }, - }), + Definition: []byte(`{"apiversion":"v1","kind":"ConfigMap","metadata":{"namespace":"foo","name":"bar"},"data":{"a":"b"}}`), + })), ExpectedOps: []opData{ {"resource-apply", 1}, }, }, { - Name: "UserResourceNop", - Input: newData().withK8sResourceReady().withResourcesReady([]cke.ResourceDefinition{ - { - Key: "DaemonSet/foo/test-daemonset", - Kind: "DaemonSet", - Namespace: "foo", - Name: "test-daemonset", - Image: "test", + Name: "UserResourceUpdate", + Input: newData().withK8sResourceReady().withResources( + []cke.ResourceDefinition{{ + Key: "Namespace/foo", + Kind: "Namespace", + Name: "foo", Revision: 2, - Definition: []byte(`{"apiVersion":"v1","kind":"DaemonSet","metadata":{"name":"test-daemonset", "namespace": "test"}}`), - }, - { - Key: "Deployment/foo/test-deployment", - Kind: "Deployment", - Namespace: "foo", - Name: "test-deployment", - Image: "test", - Revision: 1, - Definition: []byte(`{"apiVersion":"v1","kind":"Deployment","metadata":{"name":"test-deployment", "namespace": "test"}}`), - }, - }).withResourceStatus(map[string]cke.ResourceStatus{ - "DaemonSet/foo/test-daemonset": { - Annotations: map[string]string{ - cke.AnnotationResourceRevision: "2", - cke.AnnotationResourceImage: "test", - }, - Completed: false, - }, - "Deployment/foo/test-deployment": { - Annotations: map[string]string{ - cke.AnnotationResourceRevision: "1", - cke.AnnotationResourceImage: "test", - }, - Completed: false, - }, - }), + Definition: []byte(`{"apiversion":"v1","kind":"Namespace","metadata":{"name":"foo"}}`), + }}), ExpectedOps: []opData{ - {"nop", 0}, + {"resource-apply", 1}, }, }, { diff --git a/static/resources.go b/static/resources.go index cdf7e2818..56c331fcb 100644 --- a/static/resources.go +++ b/static/resources.go @@ -16,7 +16,6 @@ var Resources = []cke.ResourceDefinition{ Name: "cke-cluster-dns", Revision: 1, Image: "", - Rank: 20, Definition: []byte("apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: cke-cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/revision: \"1\"\n"), }, { @@ -26,7 +25,6 @@ var Resources = []cke.ResourceDefinition{ Name: "system:cluster-dns", Revision: 2, Image: "", - Rank: 40, Definition: []byte("\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:cluster-dns\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n annotations:\n cke.cybozu.com/revision: \"2\"\n # turn on auto-reconciliation\n # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#auto-reconciliation\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\nrules:\n - apiGroups:\n - \"\"\n resources:\n - endpoints\n - services\n - pods\n - namespaces\n verbs:\n - list\n - watch\n - apiGroups:\n - discovery.k8s.io\n resources:\n - endpointslices\n verbs:\n - list\n - watch\n"), }, { @@ -36,7 +34,6 @@ var Resources = []cke.ResourceDefinition{ Name: "system:kube-apiserver-to-kubelet", Revision: 1, Image: "", - Rank: 40, Definition: []byte("kind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:kube-apiserver-to-kubelet\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n annotations:\n cke.cybozu.com/revision: \"1\"\n # turn on auto-reconciliation\n # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#auto-reconciliation\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\nrules:\n - apiGroups: [\"\"]\n resources:\n - nodes/proxy\n - nodes/stats\n - nodes/log\n - nodes/spec\n - nodes/metrics\n verbs: [\"*\"]\n"), }, { @@ -46,7 +43,6 @@ var Resources = []cke.ResourceDefinition{ Name: "system:cluster-dns", Revision: 1, Image: "", - Rank: 50, Definition: []byte("\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:cluster-dns\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n annotations:\n cke.cybozu.com/revision: \"1\"\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:cluster-dns\nsubjects:\n- kind: ServiceAccount\n name: cke-cluster-dns\n namespace: kube-system\n"), }, { @@ -56,7 +52,6 @@ var Resources = []cke.ResourceDefinition{ Name: "system:kube-apiserver", Revision: 1, Image: "", - Rank: 50, Definition: []byte("kind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:kube-apiserver\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n annotations:\n cke.cybozu.com/revision: \"1\"\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:kube-apiserver-to-kubelet\nsubjects:\n- kind: User\n name: kubernetes\n"), }, { @@ -66,7 +61,6 @@ var Resources = []cke.ResourceDefinition{ Name: "node-dns", Revision: 4, Image: "quay.io/cybozu/unbound:1.17.1.4,quay.io/cybozu/unbound_exporter:0.4.1.5", - Rank: 3000, Definition: []byte("kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: node-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"quay.io/cybozu/unbound:1.17.1.4,quay.io/cybozu/unbound_exporter:0.4.1.5\"\n cke.cybozu.com/revision: \"4\"\nspec:\n selector:\n matchLabels:\n cke.cybozu.com/appname: node-dns\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxSurge: 35%\n maxUnavailable: 0\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: node-dns\n spec:\n priorityClassName: system-node-critical\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n - operator: Exists\n terminationGracePeriodSeconds: 1\n containers:\n - name: unbound\n image: quay.io/cybozu/unbound:1.17.1.4\n args:\n - -c\n - /etc/unbound/unbound.conf\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n add:\n - NET_BIND_SERVICE\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n livenessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n initialDelaySeconds: 1\n failureThreshold: 6\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n - name: reload\n image: quay.io/cybozu/unbound:1.17.1.4\n command:\n - /usr/local/bin/reload-unbound\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n - name: exporter\n image: quay.io/cybozu/unbound_exporter:0.4.1.5\n args:\n # must be same with the path written in /op/nodedns/nodedns.go\n - --unbound.host=unix:///var/run/unbound/unbound.sock\n - --web.reuse-port=true\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: var-run-unbound\n mountPath: /var/run/unbound\n volumes:\n - name: config-volume\n configMap:\n name: node-dns\n items:\n - key: unbound.conf\n path: unbound.conf\n - name: var-run-unbound\n emptyDir: {}\n"), }, { @@ -76,7 +70,6 @@ var Resources = []cke.ResourceDefinition{ Name: "cluster-dns", Revision: 4, Image: "quay.io/cybozu/coredns:1.10.1.1", - Rank: 3000, Definition: []byte("\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"quay.io/cybozu/coredns:1.10.1.1\"\n cke.cybozu.com/revision: \"4\"\nspec:\n replicas: 2\n strategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n selector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: cluster-dns\n k8s-app: coredns # sonobuoy requires\n annotations:\n prometheus.io/port: \"9153\"\n spec:\n priorityClassName: system-cluster-critical\n serviceAccountName: cke-cluster-dns\n tolerations:\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n - key: \"CriticalAddonsOnly\"\n operator: \"Exists\"\n - key: kubernetes.io/e2e-evict-taint-key\n operator: Exists\n # for sonobuoy https://github.com/vmware-tanzu/sonobuoy/pull/878\n containers:\n - name: coredns\n image: quay.io/cybozu/coredns:1.10.1.1\n imagePullPolicy: IfNotPresent\n resources:\n requests:\n cpu: 100m\n memory: 70Mi\n args: [ \"-conf\", \"/etc/coredns/Corefile\" ]\n lifecycle:\n preStop:\n exec:\n command: [\"sh\", \"-c\", \"sleep 5\"]\n volumeMounts:\n - name: config-volume\n mountPath: /etc/coredns\n readOnly: true\n ports:\n - containerPort: 1053\n name: dns\n protocol: UDP\n - containerPort: 1053\n name: dns-tcp\n protocol: TCP\n - containerPort: 9153\n name: metrics\n protocol: TCP\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n httpGet:\n path: /ready\n port: 8181\n scheme: HTTP\n livenessProbe:\n httpGet:\n path: /health\n port: 8080\n scheme: HTTP\n initialDelaySeconds: 60\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 5\n dnsPolicy: Default\n volumes:\n - name: config-volume\n configMap:\n name: cluster-dns\n items:\n - key: Corefile\n path: Corefile\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n topologyKey: \"kubernetes.io/hostname\"\n"), }, { @@ -86,7 +79,6 @@ var Resources = []cke.ResourceDefinition{ Name: "cluster-dns-pdb", Revision: 1, Image: "", - Rank: 3000, Definition: []byte("\napiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: cluster-dns-pdb\n namespace: kube-system\n annotations:\n cke.cybozu.com/revision: \"1\"\nspec:\n maxUnavailable: 1\n selector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n"), }, { @@ -96,7 +88,6 @@ var Resources = []cke.ResourceDefinition{ Name: "cluster-dns", Revision: 1, Image: "", - Rank: 3000, Definition: []byte("\nkind: Service\napiVersion: v1\nmetadata:\n name: cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/revision: \"1\"\n labels:\n cke.cybozu.com/appname: cluster-dns\nspec:\n selector:\n cke.cybozu.com/appname: cluster-dns\n ports:\n - name: dns\n port: 53\n targetPort: 1053\n protocol: UDP\n - name: dns-tcp\n port: 53\n targetPort: 1053\n protocol: TCP\n"), }, } diff --git a/status.go b/status.go index 8d0f83dad..5415f0914 100644 --- a/status.go +++ b/status.go @@ -48,7 +48,6 @@ type ResourceStatus struct { Annotations map[string]string // HasBeenSSA indicates that this resource has been already updated by server-side apply HasBeenSSA bool - Completed bool } // IsReady returns the cluster condition whether or not Pod can be scheduled @@ -76,8 +75,8 @@ func (s KubernetesClusterStatus) IsReady(cluster *Cluster) bool { } // SetResourceStatus sets status of the resource. -func (s KubernetesClusterStatus) SetResourceStatus(rkey string, ann map[string]string, isManaged, completed bool) { - s.ResourceStatuses[rkey] = ResourceStatus{Annotations: ann, HasBeenSSA: isManaged, Completed: completed} +func (s KubernetesClusterStatus) SetResourceStatus(rkey string, ann map[string]string, isManaged bool) { + s.ResourceStatuses[rkey] = ResourceStatus{Annotations: ann, HasBeenSSA: isManaged} } // ClusterStatus represents the working cluster status. diff --git a/storage.go b/storage.go index 1af48ebee..5895b8a85 100644 --- a/storage.go +++ b/storage.go @@ -12,7 +12,6 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/clientv3util" - "sigs.k8s.io/yaml" ) // Storage provides operations to store/retrieve CKE data in etcd. @@ -564,31 +563,12 @@ func (s Storage) GetAllResources(ctx context.Context) ([]ResourceDefinition, err return nil, errors.New("invalid resource key: " + key) } - obj := struct { - Metadata struct { - Annotations struct { - Revision int64 `json:"cke.cybozu.com/revision,string"` - Image string `json:"cke.cybozu.com/image"` - Rank uint32 `json:"cke.cybozu.com/rank,string"` - } `json:"annotations"` - } `json:"metadata"` - }{} - if err := yaml.Unmarshal(kv.Value, &obj); err != nil { - return nil, err - } - - rank, err := DecideRank(kind, namespace, obj.Metadata.Annotations.Rank) - if err != nil { - return nil, err - } - rcs = append(rcs, ResourceDefinition{ Key: key, Kind: kind, Namespace: namespace, Name: name, Revision: kv.ModRevision, - Rank: rank, Definition: kv.Value, }) } diff --git a/storage_test.go b/storage_test.go index cef2cb457..5daac1b0e 100644 --- a/storage_test.go +++ b/storage_test.go @@ -430,7 +430,7 @@ func testStorageResource(t *testing.T) { t.Error(`err != ErrNotFound,`, err) } - err = storage.SetResource(ctx, "Namespace/foo", "kind: Namespace\napiVersion: v1\nmetadata:\n name: foo\n") + err = storage.SetResource(ctx, "Namespace/foo", "bar") if err != nil { t.Fatal(err) } @@ -439,11 +439,11 @@ func testStorageResource(t *testing.T) { if err != nil { t.Fatal(err) } - if string(fooVal) != "kind: Namespace\napiVersion: v1\nmetadata:\n name: foo\n" { - t.Error(`string(fooVal) != "kind: Namespace\napiVersion: v1\nmetadata:\n name: foo\n",`, string(fooVal)) + if string(fooVal) != "bar" { + t.Error(`string(fooVal) != "bar",`, string(fooVal)) } - err = storage.SetResource(ctx, "Pod/foo/pod1", "kind: Pod\napiVersion: v1\nmetadata:\n name: pod1\n namespace: foo\n") + err = storage.SetResource(ctx, "Pod/foo/pod1", "test") if err != nil { t.Fatal(err) } @@ -457,22 +457,12 @@ func testStorageResource(t *testing.T) { t.Error("unexpected list result:", cmp.Diff(expectedKeys, keys)) } - err = storage.SetResource(ctx, "ConfigMap/foo/conf1", "kind: ConfigMap\napiVersion: v1\nmetadata:\n name: conf1\n namespace: foo\n") + err = storage.SetResource(ctx, "ConfigMap/foo/conf1", "test") if err != nil { t.Fatal(err) } - err = storage.SetResource(ctx, "ServiceAccount/foo/sa1", "kind: ConfigMap\napiVersion: v1\nmetadata:\n name: sa1\n namespace: foo\n") - if err != nil { - t.Fatal(err) - } - - err = storage.SetResource(ctx, "Deployment/foo/deploy1", "\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"quay.io/cybozu/coredns:1.10.0.1\"\n cke.cybozu.com/revision: \"4\"\n cke.cybozu.com/rank: \"2200\"\nspec:\n") - if err != nil { - t.Fatal(err) - } - - err = storage.SetResource(ctx, "DaemonSet/foo/ds1", "kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: node-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"quay.io/cybozu/unbound:1.17.1.1,quay.io/cybozu/unbound_exporter:0.4.1.4\"\n cke.cybozu.com/revision: \"4\"\n cke.cybozu.com/rank: \"2100\"\nspec:\n") + err = storage.SetResource(ctx, "ServiceAccount/foo/sa1", "test") if err != nil { t.Fatal(err) } @@ -482,7 +472,7 @@ func testStorageResource(t *testing.T) { t.Fatal(err) } - expected := []string{"Namespace/foo", "ServiceAccount/foo/sa1", "ConfigMap/foo/conf1", "DaemonSet/foo/ds1", "Deployment/foo/deploy1", "Pod/foo/pod1"} + expected := []string{"Namespace/foo", "ServiceAccount/foo/sa1", "ConfigMap/foo/conf1", "Pod/foo/pod1"} actual := make([]string, len(resources)) for i, r := range resources { actual[i] = r.Key