From 02fb62c4aab152426af2b7f0a9fea68e00fe017c Mon Sep 17 00:00:00 2001 From: terasihma Date: Wed, 17 May 2023 06:01:59 +0000 Subject: [PATCH] check observedGeneration and output Rank by compile_resources Signed-off-by: terasihma --- op/status.go | 26 +++++++++++++++++++++++--- pkg/compile_resources/main.go | 1 + resource.go | 2 +- static/resources.go | 9 +++++++++ 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/op/status.go b/op/status.go index 2c62896d..451d433b 100644 --- a/op/status.go +++ b/op/status.go @@ -437,6 +437,15 @@ func GetKubernetesClusterStatus(ctx context.Context, inf cke.Infrastructure, n * return cke.KubernetesClusterStatus{}, err } if obj.GroupVersionKind().Kind == "DaemonSet" { + generation, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "metadata", "generation") + if err != nil { + return cke.KubernetesClusterStatus{}, err + } + observedGeneration, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "observedGeneration") + if err != nil { + return cke.KubernetesClusterStatus{}, err + } + desired, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "desiredNumberScheduled") if err != nil { return cke.KubernetesClusterStatus{}, err @@ -449,8 +458,16 @@ func GetKubernetesClusterStatus(ctx context.Context, inf cke.Infrastructure, n * if err != nil { return cke.KubernetesClusterStatus{}, err } - s.SetResourceStatus(res.Key, obj.GetAnnotations(), len(obj.GetManagedFields()) != 0, objStatus(desired, updated, available)) + s.SetResourceStatus(res.Key, obj.GetAnnotations(), len(obj.GetManagedFields()) != 0, objStatus(generation, observedGeneration, desired, updated, available)) } else if obj.GroupVersionKind().Kind == "Deployment" { + generation, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "metadata", "generation") + if err != nil { + return cke.KubernetesClusterStatus{}, err + } + observedGeneration, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "observedGeneration") + if err != nil { + return cke.KubernetesClusterStatus{}, err + } desired, _, err := unstructured.NestedInt64(obj.UnstructuredContent(), "status", "readyReplicas") if err != nil { return cke.KubernetesClusterStatus{}, err @@ -463,7 +480,7 @@ func GetKubernetesClusterStatus(ctx context.Context, inf cke.Infrastructure, n * if err != nil { return cke.KubernetesClusterStatus{}, err } - s.SetResourceStatus(res.Key, obj.GetAnnotations(), len(obj.GetManagedFields()) != 0, objStatus(desired, updated, available)) + s.SetResourceStatus(res.Key, obj.GetAnnotations(), len(obj.GetManagedFields()) != 0, objStatus(generation, observedGeneration, desired, updated, available)) } else { s.SetResourceStatus(res.Key, obj.GetAnnotations(), len(obj.GetManagedFields()) != 0, true) } @@ -472,7 +489,10 @@ func GetKubernetesClusterStatus(ctx context.Context, inf cke.Infrastructure, n * return s, nil } -func objStatus(desired, updated, available int64) bool { +func objStatus(generation, observedGeneration, desired, updated, available int64) bool { + if generation > observedGeneration { + return false + } // If we get the status immediately after applying the resource, the value of desired may be 0. // In this case, we need to return false. if desired == 0 { diff --git a/pkg/compile_resources/main.go b/pkg/compile_resources/main.go index 8a4429b1..45bb0985 100644 --- a/pkg/compile_resources/main.go +++ b/pkg/compile_resources/main.go @@ -143,6 +143,7 @@ var Resources = []cke.ResourceDefinition{ Name: {{ printf "%q" .Name }}, Revision: {{ .Revision }}, Image: {{ printf "%q" .Image }}, + Rank: {{ .Rank }}, Definition: []byte({{ printf "%q" .Definition }}), }, {{ end -}} diff --git a/resource.go b/resource.go index dd6c0724..d9a0197d 100644 --- a/resource.go +++ b/resource.go @@ -42,7 +42,7 @@ const ( // rank const ( RankNamespace = 10 - RankServiceAccount = 20 + RankServiceAccount = 20 // ServiceAccount is namespace scoped RankCustomResourceDefinition = 30 RankClusterRole = 40 RankClusterRoleBinding = 50 diff --git a/static/resources.go b/static/resources.go index 0b2d7c67..a0daeb44 100644 --- a/static/resources.go +++ b/static/resources.go @@ -16,6 +16,7 @@ var Resources = []cke.ResourceDefinition{ Name: "cke-cluster-dns", Revision: 1, Image: "", + Rank: 20, Definition: []byte("apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: cke-cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/revision: \"1\"\n"), }, { @@ -25,6 +26,7 @@ var Resources = []cke.ResourceDefinition{ Name: "system:cluster-dns", Revision: 2, Image: "", + Rank: 40, Definition: []byte("\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:cluster-dns\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n annotations:\n cke.cybozu.com/revision: \"2\"\n # turn on auto-reconciliation\n # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#auto-reconciliation\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\nrules:\n - apiGroups:\n - \"\"\n resources:\n - endpoints\n - services\n - pods\n - namespaces\n verbs:\n - list\n - watch\n - apiGroups:\n - discovery.k8s.io\n resources:\n - endpointslices\n verbs:\n - list\n - watch\n"), }, { @@ -34,6 +36,7 @@ var Resources = []cke.ResourceDefinition{ Name: "system:kube-apiserver-to-kubelet", Revision: 1, Image: "", + Rank: 40, Definition: []byte("kind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:kube-apiserver-to-kubelet\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n annotations:\n cke.cybozu.com/revision: \"1\"\n # turn on auto-reconciliation\n # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#auto-reconciliation\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\nrules:\n - apiGroups: [\"\"]\n resources:\n - nodes/proxy\n - nodes/stats\n - nodes/log\n - nodes/spec\n - nodes/metrics\n verbs: [\"*\"]\n"), }, { @@ -43,6 +46,7 @@ var Resources = []cke.ResourceDefinition{ Name: "system:cluster-dns", Revision: 1, Image: "", + Rank: 50, Definition: []byte("\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:cluster-dns\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n annotations:\n cke.cybozu.com/revision: \"1\"\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:cluster-dns\nsubjects:\n- kind: ServiceAccount\n name: cke-cluster-dns\n namespace: kube-system\n"), }, { @@ -52,6 +56,7 @@ var Resources = []cke.ResourceDefinition{ Name: "system:kube-apiserver", Revision: 1, Image: "", + Rank: 50, Definition: []byte("kind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:kube-apiserver\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n annotations:\n cke.cybozu.com/revision: \"1\"\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:kube-apiserver-to-kubelet\nsubjects:\n- kind: User\n name: kubernetes\n"), }, { @@ -61,6 +66,7 @@ var Resources = []cke.ResourceDefinition{ Name: "node-dns", Revision: 4, Image: "quay.io/cybozu/unbound:1.17.1.3,quay.io/cybozu/unbound_exporter:0.4.1.5", + Rank: 3000, Definition: []byte("kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: node-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"quay.io/cybozu/unbound:1.17.1.3,quay.io/cybozu/unbound_exporter:0.4.1.5\"\n cke.cybozu.com/revision: \"4\"\nspec:\n selector:\n matchLabels:\n cke.cybozu.com/appname: node-dns\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxSurge: 35%\n maxUnavailable: 0\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: node-dns\n spec:\n priorityClassName: system-node-critical\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n - operator: Exists\n terminationGracePeriodSeconds: 1\n containers:\n - name: unbound\n image: quay.io/cybozu/unbound:1.17.1.3\n args:\n - -c\n - /etc/unbound/unbound.conf\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n add:\n - NET_BIND_SERVICE\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n livenessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n initialDelaySeconds: 1\n failureThreshold: 6\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n - name: reload\n image: quay.io/cybozu/unbound:1.17.1.3\n command:\n - /usr/local/bin/reload-unbound\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n - name: exporter\n image: quay.io/cybozu/unbound_exporter:0.4.1.5\n args:\n # must be same with the path written in /op/nodedns/nodedns.go\n - --unbound.host=unix:///var/run/unbound/unbound.sock\n - --web.reuse-port=true\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: var-run-unbound\n mountPath: /var/run/unbound\n volumes:\n - name: config-volume\n configMap:\n name: node-dns\n items:\n - key: unbound.conf\n path: unbound.conf\n - name: var-run-unbound\n emptyDir: {}\n"), }, { @@ -70,6 +76,7 @@ var Resources = []cke.ResourceDefinition{ Name: "cluster-dns", Revision: 4, Image: "quay.io/cybozu/coredns:1.10.0.2", + Rank: 3000, Definition: []byte("\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"quay.io/cybozu/coredns:1.10.0.2\"\n cke.cybozu.com/revision: \"4\"\nspec:\n replicas: 2\n strategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n selector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: cluster-dns\n k8s-app: coredns # sonobuoy requires\n annotations:\n prometheus.io/port: \"9153\"\n spec:\n priorityClassName: system-cluster-critical\n serviceAccountName: cke-cluster-dns\n tolerations:\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n - key: \"CriticalAddonsOnly\"\n operator: \"Exists\"\n - key: kubernetes.io/e2e-evict-taint-key\n operator: Exists\n # for sonobuoy https://github.com/vmware-tanzu/sonobuoy/pull/878\n containers:\n - name: coredns\n image: quay.io/cybozu/coredns:1.10.0.2\n imagePullPolicy: IfNotPresent\n resources:\n requests:\n cpu: 100m\n memory: 70Mi\n args: [ \"-conf\", \"/etc/coredns/Corefile\" ]\n lifecycle:\n preStop:\n exec:\n command: [\"sh\", \"-c\", \"sleep 5\"]\n volumeMounts:\n - name: config-volume\n mountPath: /etc/coredns\n readOnly: true\n ports:\n - containerPort: 1053\n name: dns\n protocol: UDP\n - containerPort: 1053\n name: dns-tcp\n protocol: TCP\n - containerPort: 9153\n name: metrics\n protocol: TCP\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n httpGet:\n path: /ready\n port: 8181\n scheme: HTTP\n livenessProbe:\n httpGet:\n path: /health\n port: 8080\n scheme: HTTP\n initialDelaySeconds: 60\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 5\n dnsPolicy: Default\n volumes:\n - name: config-volume\n configMap:\n name: cluster-dns\n items:\n - key: Corefile\n path: Corefile\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n topologyKey: \"kubernetes.io/hostname\"\n"), }, { @@ -79,6 +86,7 @@ var Resources = []cke.ResourceDefinition{ Name: "cluster-dns-pdb", Revision: 1, Image: "", + Rank: 3000, Definition: []byte("\napiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: cluster-dns-pdb\n namespace: kube-system\n annotations:\n cke.cybozu.com/revision: \"1\"\nspec:\n maxUnavailable: 1\n selector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n"), }, { @@ -88,6 +96,7 @@ var Resources = []cke.ResourceDefinition{ Name: "cluster-dns", Revision: 1, Image: "", + Rank: 3000, Definition: []byte("\nkind: Service\napiVersion: v1\nmetadata:\n name: cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/revision: \"1\"\n labels:\n cke.cybozu.com/appname: cluster-dns\nspec:\n selector:\n cke.cybozu.com/appname: cluster-dns\n ports:\n - name: dns\n port: 53\n targetPort: 1053\n protocol: UDP\n - name: dns-tcp\n port: 53\n targetPort: 1053\n protocol: TCP\n"), }, }