From 8fa64137ab31a415357394ca3ab1b5e61ece2fc7 Mon Sep 17 00:00:00 2001 From: "mingzhou.swx" Date: Fri, 20 May 2022 11:52:42 +0800 Subject: [PATCH] add lifecycle mark pod not ready feature Signed-off-by: mingzhou.swx --- apis/apps/pub/lifecycle.go | 5 ++ apis/apps/pub/pod_readiness_gate.go | 7 ++ .../crd/bases/apps.kruise.io_clonesets.yaml | 14 +++ .../crd/bases/apps.kruise.io_daemonsets.yaml | 14 +++ .../bases/apps.kruise.io_statefulsets.yaml | 14 +++ .../apps.kruise.io_uniteddeployments.yaml | 28 ++++++ pkg/controller/cloneset/cloneset_status.go | 3 +- .../cloneset/sync/cloneset_scale.go | 15 +++- .../cloneset/sync/cloneset_sync_utils.go | 8 +- .../cloneset/sync/cloneset_update.go | 13 ++- .../crr_controller.go | 13 +-- .../daemonset/daemonset_controller.go | 4 +- .../statefulset/stateful_set_control.go | 16 ++-- pkg/util/lifecycle/lifecycle_utils.go | 85 +++++++++++++++++-- pkg/util/podreadiness/pod_readiness.go | 75 ++++++++++++++++ pkg/util/podreadiness/pod_readiness_utils.go | 84 ++++++++---------- .../podreadiness/pod_readiness_utils_test.go | 26 +++--- 17 files changed, 335 insertions(+), 89 deletions(-) create mode 100644 pkg/util/podreadiness/pod_readiness.go diff --git a/apis/apps/pub/lifecycle.go b/apis/apps/pub/lifecycle.go index 928f81fbf4..39c8ae33ce 100644 --- a/apis/apps/pub/lifecycle.go +++ b/apis/apps/pub/lifecycle.go @@ -40,4 +40,9 @@ type Lifecycle struct { type LifecycleHook struct { LabelsHandler map[string]string `json:"labelsHandler,omitempty"` FinalizersHandler []string `json:"finalizersHandler,omitempty"` + // MarkPodNotReady = true means: + // - Pod will be set to 'NotReady' at preparingDelete/preparingUpdate state. + // - Pod will be restored to 'Ready' at Updated state if it was set to 'NotReady' at preparingUpdate state. + // Default to false. + MarkPodNotReady bool `json:"markPodNotReady,omitempty"` } diff --git a/apis/apps/pub/pod_readiness_gate.go b/apis/apps/pub/pod_readiness_gate.go index 4d47c7fb3a..2dbe930d65 100644 --- a/apis/apps/pub/pod_readiness_gate.go +++ b/apis/apps/pub/pod_readiness_gate.go @@ -19,5 +19,12 @@ package pub import v1 "k8s.io/api/core/v1" const ( + // KruisePodReadyConditionType can support multiple writers, such as: + // - ContainerRecreateRequest; + // - Workload controller, including CloneSet, Advanced StatefulSet, Advanced Daemonset. + // + // If its corresponding condition status was set to "False" by multiple writers, + // the condition status will be considered as "True" only when all these writers + // set it to "True". KruisePodReadyConditionType v1.PodConditionType = "KruisePodReady" ) diff --git a/config/crd/bases/apps.kruise.io_clonesets.yaml b/config/crd/bases/apps.kruise.io_clonesets.yaml index 2c77a0ca24..da2fb66cdc 100644 --- a/config/crd/bases/apps.kruise.io_clonesets.yaml +++ b/config/crd/bases/apps.kruise.io_clonesets.yaml @@ -97,6 +97,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - Pod will be + set to ''NotReady'' at preparingDelete/preparingUpdate state. + - Pod will be restored to ''Ready'' at Updated state if + it was set to ''NotReady'' at preparingUpdate state. Default + to false.' + type: boolean type: object preDelete: description: PreDelete is the hook before Pod to be deleted. @@ -109,6 +116,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - Pod will be + set to ''NotReady'' at preparingDelete/preparingUpdate state. + - Pod will be restored to ''Ready'' at Updated state if + it was set to ''NotReady'' at preparingUpdate state. Default + to false.' + type: boolean type: object type: object minReadySeconds: diff --git a/config/crd/bases/apps.kruise.io_daemonsets.yaml b/config/crd/bases/apps.kruise.io_daemonsets.yaml index f7301b9811..026394d23b 100644 --- a/config/crd/bases/apps.kruise.io_daemonsets.yaml +++ b/config/crd/bases/apps.kruise.io_daemonsets.yaml @@ -93,6 +93,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - Pod will be + set to ''NotReady'' at preparingDelete/preparingUpdate state. + - Pod will be restored to ''Ready'' at Updated state if + it was set to ''NotReady'' at preparingUpdate state. Default + to false.' + type: boolean type: object preDelete: description: PreDelete is the hook before Pod to be deleted. @@ -105,6 +112,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - Pod will be + set to ''NotReady'' at preparingDelete/preparingUpdate state. + - Pod will be restored to ''Ready'' at Updated state if + it was set to ''NotReady'' at preparingUpdate state. Default + to false.' + type: boolean type: object type: object minReadySeconds: diff --git a/config/crd/bases/apps.kruise.io_statefulsets.yaml b/config/crd/bases/apps.kruise.io_statefulsets.yaml index 72efc0d4be..2250fdf00b 100644 --- a/config/crd/bases/apps.kruise.io_statefulsets.yaml +++ b/config/crd/bases/apps.kruise.io_statefulsets.yaml @@ -514,6 +514,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - Pod will be + set to ''NotReady'' at preparingDelete/preparingUpdate state. + - Pod will be restored to ''Ready'' at Updated state if + it was set to ''NotReady'' at preparingUpdate state. Default + to false.' + type: boolean type: object preDelete: description: PreDelete is the hook before Pod to be deleted. @@ -526,6 +533,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - Pod will be + set to ''NotReady'' at preparingDelete/preparingUpdate state. + - Pod will be restored to ''Ready'' at Updated state if + it was set to ''NotReady'' at preparingUpdate state. Default + to false.' + type: boolean type: object type: object persistentVolumeClaimRetentionPolicy: diff --git a/config/crd/bases/apps.kruise.io_uniteddeployments.yaml b/config/crd/bases/apps.kruise.io_uniteddeployments.yaml index 9d394082c9..a274937b18 100644 --- a/config/crd/bases/apps.kruise.io_uniteddeployments.yaml +++ b/config/crd/bases/apps.kruise.io_uniteddeployments.yaml @@ -145,6 +145,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - + Pod will be set to ''NotReady'' at preparingDelete/preparingUpdate + state. - Pod will be restored to ''Ready'' at + Updated state if it was set to ''NotReady'' + at preparingUpdate state. Default to false.' + type: boolean type: object preDelete: description: PreDelete is the hook before Pod to be @@ -158,6 +165,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - + Pod will be set to ''NotReady'' at preparingDelete/preparingUpdate + state. - Pod will be restored to ''Ready'' at + Updated state if it was set to ''NotReady'' + at preparingUpdate state. Default to false.' + type: boolean type: object type: object persistentVolumeClaimRetentionPolicy: @@ -546,6 +560,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - + Pod will be set to ''NotReady'' at preparingDelete/preparingUpdate + state. - Pod will be restored to ''Ready'' at + Updated state if it was set to ''NotReady'' + at preparingUpdate state. Default to false.' + type: boolean type: object preDelete: description: PreDelete is the hook before Pod to be @@ -559,6 +580,13 @@ spec: additionalProperties: type: string type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - + Pod will be set to ''NotReady'' at preparingDelete/preparingUpdate + state. - Pod will be restored to ''Ready'' at + Updated state if it was set to ''NotReady'' + at preparingUpdate state. Default to false.' + type: boolean type: object type: object minReadySeconds: diff --git a/pkg/controller/cloneset/cloneset_status.go b/pkg/controller/cloneset/cloneset_status.go index 6de8c7a6a2..abc7fa2c50 100644 --- a/pkg/controller/cloneset/cloneset_status.go +++ b/pkg/controller/cloneset/cloneset_status.go @@ -21,6 +21,7 @@ import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" clonesetcore "github.com/openkruise/kruise/pkg/controller/cloneset/core" + "github.com/openkruise/kruise/pkg/controller/cloneset/sync" clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils" "github.com/openkruise/kruise/pkg/util" v1 "k8s.io/api/core/v1" @@ -88,7 +89,7 @@ func (r *realStatusUpdater) calculateStatus(cs *appsv1alpha1.CloneSet, newStatus if coreControl.IsPodUpdateReady(pod, 0) { newStatus.ReadyReplicas++ } - if coreControl.IsPodUpdateReady(pod, cs.Spec.MinReadySeconds) { + if sync.IsPodAvailable(coreControl, pod, cs.Spec.MinReadySeconds) { newStatus.AvailableReplicas++ } if clonesetutils.EqualToRevisionHash("", pod, newStatus.UpdateRevision) { diff --git a/pkg/controller/cloneset/sync/cloneset_scale.go b/pkg/controller/cloneset/sync/cloneset_scale.go index e5a4fb071a..b696822d4d 100644 --- a/pkg/controller/cloneset/sync/cloneset_scale.go +++ b/pkg/controller/cloneset/sync/cloneset_scale.go @@ -165,6 +165,14 @@ func (r *realControl) Scale( } func (r *realControl) managePreparingDelete(cs *appsv1alpha1.CloneSet, pods, podsInPreDelete []*v1.Pod, numToDelete int) (bool, error) { + // We do not allow regret once the pod enter PreparingDelete state if MarkPodNotReady is set. + // Actually, there is a bug cased by this transformation from PreparingDelete to Normal, + // i.e., Lifecycle Updated Hook may be lost if the pod was transformed from Updating state + // to PreparingDelete. + if lifecycle.IsLifecycleMarkPodNotReady(cs.Spec.Lifecycle) { + return false, nil + } + diff := int(*cs.Spec.Replicas) - len(pods) + numToDelete var modified bool for _, pod := range podsInPreDelete { @@ -177,7 +185,7 @@ func (r *realControl) managePreparingDelete(cs *appsv1alpha1.CloneSet, pods, pod klog.V(3).Infof("CloneSet %s patch pod %s lifecycle from PreparingDelete to Normal", clonesetutils.GetControllerKey(cs), pod.Name) - if updated, gotPod, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStateNormal); err != nil { + if updated, gotPod, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStateNormal, false); err != nil { return modified, err } else if updated { modified = true @@ -270,7 +278,8 @@ func (r *realControl) deletePods(cs *appsv1alpha1.CloneSet, podsToDelete []*v1.P var modified bool for _, pod := range podsToDelete { if cs.Spec.Lifecycle != nil && lifecycle.IsPodHooked(cs.Spec.Lifecycle.PreDelete, pod) { - if updated, gotPod, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingDelete); err != nil { + markPodNotReady := cs.Spec.Lifecycle.PreDelete.MarkPodNotReady + if updated, gotPod, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingDelete, markPodNotReady); err != nil { return false, err } else if updated { klog.V(3).Infof("CloneSet %s scaling update pod %s lifecycle to PreparingDelete", @@ -383,7 +392,7 @@ func (r *realControl) choosePodsToDelete(cs *appsv1alpha1.CloneSet, totalDiff in Pods: pods, Ranker: ranker, AvailableFunc: func(pod *v1.Pod) bool { - return isPodAvailable(coreControl, pod, cs.Spec.MinReadySeconds) + return IsPodAvailable(coreControl, pod, cs.Spec.MinReadySeconds) }, }) } else if diff > len(pods) { diff --git a/pkg/controller/cloneset/sync/cloneset_sync_utils.go b/pkg/controller/cloneset/sync/cloneset_sync_utils.go index 3b54c92d65..493228c557 100644 --- a/pkg/controller/cloneset/sync/cloneset_sync_utils.go +++ b/pkg/controller/cloneset/sync/cloneset_sync_utils.go @@ -122,7 +122,7 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu if isSpecifiedDelete(cs, p) { toDeleteNewRevisionCount++ - } else if !isPodAvailable(coreControl, p, cs.Spec.MinReadySeconds) { + } else if !IsPodAvailable(coreControl, p, cs.Spec.MinReadySeconds) { unavailableNewRevisionCount++ } } @@ -138,7 +138,7 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu if isSpecifiedDelete(cs, p) { toDeleteOldRevisionCount++ - } else if !isPodAvailable(coreControl, p, cs.Spec.MinReadySeconds) { + } else if !IsPodAvailable(coreControl, p, cs.Spec.MinReadySeconds) { unavailableOldRevisionCount++ } } @@ -235,10 +235,10 @@ func isSpecifiedDelete(cs *appsv1alpha1.CloneSet, pod *v1.Pod) bool { } func isPodReady(coreControl clonesetcore.Control, pod *v1.Pod) bool { - return isPodAvailable(coreControl, pod, 0) + return IsPodAvailable(coreControl, pod, 0) } -func isPodAvailable(coreControl clonesetcore.Control, pod *v1.Pod, minReadySeconds int32) bool { +func IsPodAvailable(coreControl clonesetcore.Control, pod *v1.Pod, minReadySeconds int32) bool { state := lifecycle.GetPodLifecycleState(pod) if state != "" && state != appspub.LifecycleStateNormal { return false diff --git a/pkg/controller/cloneset/sync/cloneset_update.go b/pkg/controller/cloneset/sync/cloneset_update.go index 109b8aa6ac..750ad6a202 100644 --- a/pkg/controller/cloneset/sync/cloneset_update.go +++ b/pkg/controller/cloneset/sync/cloneset_update.go @@ -194,7 +194,11 @@ func (c *realControl) refreshPodState(cs *appsv1alpha1.CloneSet, coreControl clo } if state != "" { - if updated, gotPod, err := c.lifecycleControl.UpdatePodLifecycle(pod, state); err != nil { + var markPodNotReady bool + if cs.Spec.Lifecycle != nil && cs.Spec.Lifecycle.InPlaceUpdate != nil { + markPodNotReady = cs.Spec.Lifecycle.InPlaceUpdate.MarkPodNotReady + } + if updated, gotPod, err := c.lifecycleControl.UpdatePodLifecycle(pod, state, markPodNotReady); err != nil { return false, 0, err } else if updated { clonesetutils.ResourceVersionExpectations.Expect(gotPod) @@ -245,7 +249,8 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc var updated bool var gotPod *v1.Pod if cs.Spec.Lifecycle != nil && lifecycle.IsPodHooked(cs.Spec.Lifecycle.InPlaceUpdate, pod) { - if updated, gotPod, err = c.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingUpdate); err == nil && updated { + markPodNotReady := cs.Spec.Lifecycle.InPlaceUpdate.MarkPodNotReady + if updated, gotPod, err = c.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingUpdate, markPodNotReady); err == nil && updated { clonesetutils.ResourceVersionExpectations.Expect(gotPod) klog.V(3).Infof("CloneSet %s update pod %s lifecycle to PreparingUpdate", clonesetutils.GetControllerKey(cs), pod.Name) @@ -344,7 +349,7 @@ func limitUpdateIndexes(coreControl clonesetcore.Control, minReadySeconds int32, var unavailableCount, targetRevisionUnavailableCount, canUpdateCount int for _, p := range pods { - if !isPodAvailable(coreControl, p, minReadySeconds) { + if !IsPodAvailable(coreControl, p, minReadySeconds) { unavailableCount++ if clonesetutils.EqualToRevisionHash("", p, targetRevisionHash) { targetRevisionUnavailableCount++ @@ -359,7 +364,7 @@ func limitUpdateIndexes(coreControl clonesetcore.Control, minReadySeconds int32, // Make sure unavailable pods in all revisions should not be more than maxUnavailable. // Note that update an old pod that already be unavailable will not increase the unavailable number. - if isPodAvailable(coreControl, pods[i], minReadySeconds) { + if IsPodAvailable(coreControl, pods[i], minReadySeconds) { if unavailableCount >= diffRes.updateMaxUnavailable { break } diff --git a/pkg/controller/containerrecreaterequest/crr_controller.go b/pkg/controller/containerrecreaterequest/crr_controller.go index 40824089c8..f99fa11414 100644 --- a/pkg/controller/containerrecreaterequest/crr_controller.go +++ b/pkg/controller/containerrecreaterequest/crr_controller.go @@ -70,9 +70,11 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) *ReconcileContainerRecreateRequest { + cli := util.NewClientFromManager(mgr, "containerrecreaterequest-controller") return &ReconcileContainerRecreateRequest{ - Client: util.NewClientFromManager(mgr, "containerrecreaterequest-controller"), - clock: clock.RealClock{}, + Client: cli, + clock: clock.RealClock{}, + podReadinessControl: utilpodreadiness.New(cli), } } @@ -104,7 +106,8 @@ var _ reconcile.Reconciler = &ReconcileContainerRecreateRequest{} // ReconcileContainerRecreateRequest reconciles a ContainerRecreateRequest object type ReconcileContainerRecreateRequest struct { client.Client - clock clock.Clock + clock clock.Clock + podReadinessControl utilpodreadiness.Interface } // +kubebuilder:rbac:groups=apps.kruise.io,resources=containerrecreaterequests,verbs=get;list;watch;create;update;patch;delete @@ -273,7 +276,7 @@ func (r *ReconcileContainerRecreateRequest) acquirePodNotReady(crr *appsv1alpha1 } } - err := utilpodreadiness.AddNotReadyKey(r.Client, pod, getReadinessMessage(crr)) + err := r.podReadinessControl.AddNotReadyKey(pod, getReadinessMessage(crr)) if err != nil { return fmt.Errorf("add Pod not ready error: %v", err) } @@ -287,7 +290,7 @@ func (r *ReconcileContainerRecreateRequest) acquirePodNotReady(crr *appsv1alpha1 func (r *ReconcileContainerRecreateRequest) releasePodNotReady(crr *appsv1alpha1.ContainerRecreateRequest, pod *v1.Pod) error { if pod != nil && pod.DeletionTimestamp == nil && utilpodreadiness.ContainsReadinessGate(pod) { - err := utilpodreadiness.RemoveNotReadyKey(r.Client, pod, getReadinessMessage(crr)) + err := r.podReadinessControl.RemoveNotReadyKey(pod, getReadinessMessage(crr)) if err != nil { return fmt.Errorf("remove Pod not ready error: %v", err) } diff --git a/pkg/controller/daemonset/daemonset_controller.go b/pkg/controller/daemonset/daemonset_controller.go index 7ad78b9d4b..27e732f21d 100644 --- a/pkg/controller/daemonset/daemonset_controller.go +++ b/pkg/controller/daemonset/daemonset_controller.go @@ -755,7 +755,8 @@ func (dsc *ReconcileDaemonSet) syncWithPreparingDelete(ds *appsv1alpha1.DaemonSe podsCanDelete = append(podsCanDelete, podName) continue } - if updated, gotPod, err := dsc.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingDelete); err != nil { + markPodNotReady := ds.Spec.Lifecycle.PreDelete.MarkPodNotReady + if updated, gotPod, err := dsc.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingDelete, markPodNotReady); err != nil { return nil, err } else if updated { klog.V(3).Infof("DaemonSet %s/%s has marked Pod %s as PreparingDelete", ds.Namespace, ds.Name, podName) @@ -989,7 +990,6 @@ func (dsc *ReconcileDaemonSet) refreshUpdateStates(ds *appsv1alpha1.DaemonSet) e opts := &inplaceupdate.UpdateOptions{} opts = inplaceupdate.SetOptionsDefaults(opts) - for _, pod := range pods { if dsc.inplaceControl == nil { continue diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index d8c578533d..2f5e213328 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -493,7 +493,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( set.Namespace, set.Name, replicas[i].Name) - if err := ssc.podControl.DeleteStatefulPod(set, replicas[i]); err != nil { + if _, err := ssc.deletePod(set, replicas[i]); err != nil { return &status, err } if getPodRevision(replicas[i]) == currentRevision.Name { @@ -740,7 +740,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( set.Namespace, set.Name, replicas[target].Name) - if err := ssc.podControl.DeleteStatefulPod(set, replicas[target]); err != nil { + if _, err := ssc.deletePod(set, replicas[target]); err != nil { return &status, err } } @@ -790,7 +790,8 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( func (ssc *defaultStatefulSetControl) deletePod(set *appsv1beta1.StatefulSet, pod *v1.Pod) (bool, error) { if set.Spec.Lifecycle != nil && lifecycle.IsPodHooked(set.Spec.Lifecycle.PreDelete, pod) { - if updated, _, err := ssc.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingDelete); err != nil { + markPodNotReady := set.Spec.Lifecycle.PreDelete.MarkPodNotReady + if updated, _, err := ssc.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingDelete, markPodNotReady); err != nil { return false, err } else if updated { klog.V(3).Infof("StatefulSet %s scaling update pod %s lifecycle to PreparingDelete", @@ -842,7 +843,11 @@ func (ssc *defaultStatefulSetControl) refreshPodState(set *appsv1beta1.StatefulS } if state != "" { - if updated, _, err := ssc.lifecycleControl.UpdatePodLifecycle(pod, state); err != nil { + var markPodNotReady bool + if set.Spec.Lifecycle != nil && set.Spec.Lifecycle.InPlaceUpdate != nil { + markPodNotReady = set.Spec.Lifecycle.InPlaceUpdate.MarkPodNotReady + } + if updated, _, err := ssc.lifecycleControl.UpdatePodLifecycle(pod, state, markPodNotReady); err != nil { return false, 0, err } else if updated { klog.V(3).Infof("AdvancedStatefulSet %s update pod %s lifecycle to %s", @@ -886,7 +891,8 @@ func (ssc *defaultStatefulSetControl) inPlaceUpdatePod( var err error var updated bool if set.Spec.Lifecycle != nil && lifecycle.IsPodHooked(set.Spec.Lifecycle.InPlaceUpdate, pod) { - if updated, _, err = ssc.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingUpdate); err == nil && updated { + markPodNotReady := set.Spec.Lifecycle.InPlaceUpdate.MarkPodNotReady + if updated, _, err = ssc.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingUpdate, markPodNotReady); err == nil && updated { klog.V(3).Infof("StatefulSet %s updated pod %s lifecycle to PreparingUpdate", getStatefulSetKey(set), pod.Name) } diff --git a/pkg/util/lifecycle/lifecycle_utils.go b/pkg/util/lifecycle/lifecycle_utils.go index ab7d25974d..8a16536ea4 100644 --- a/pkg/util/lifecycle/lifecycle_utils.go +++ b/pkg/util/lifecycle/lifecycle_utils.go @@ -22,42 +22,80 @@ import ( "time" appspub "github.com/openkruise/kruise/apis/apps/pub" + "github.com/openkruise/kruise/pkg/util/podadapter" + "github.com/openkruise/kruise/pkg/util/podreadiness" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) - "github.com/openkruise/kruise/pkg/util/podadapter" +const ( + // these keys for MarkPodNotReady Policy of pod lifecycle + preparingDeleteHookKey = "preDeleteHook" + preparingUpdateHookKey = "preUpdateHook" ) +func newPodReadinessControl(adp podadapter.Adapter) podreadiness.Interface { + return podreadiness.NewForAdapter(adp) +} + // Interface for managing pods lifecycle. type Interface interface { - UpdatePodLifecycle(pod *v1.Pod, state appspub.LifecycleStateType) (bool, *v1.Pod, error) + UpdatePodLifecycle(pod *v1.Pod, state appspub.LifecycleStateType, markPodNotReady bool) (bool, *v1.Pod, error) UpdatePodLifecycleWithHandler(pod *v1.Pod, state appspub.LifecycleStateType, inPlaceUpdateHandler *appspub.LifecycleHook) (bool, *v1.Pod, error) } type realControl struct { - adp podadapter.Adapter + adp podadapter.Adapter + podReadinessControl podreadiness.Interface } func New(c client.Client) Interface { - return &realControl{adp: &podadapter.AdapterRuntimeClient{Client: c}} + adp := &podadapter.AdapterRuntimeClient{Client: c} + return &realControl{ + adp: adp, + podReadinessControl: newPodReadinessControl(adp), + } } func NewForTypedClient(c clientset.Interface) Interface { - return &realControl{adp: &podadapter.AdapterTypedClient{Client: c}} + adp := &podadapter.AdapterTypedClient{Client: c} + return &realControl{ + adp: adp, + podReadinessControl: newPodReadinessControl(adp), + } } func NewForInformer(informer coreinformers.PodInformer) Interface { - return &realControl{adp: &podadapter.AdapterInformer{PodInformer: informer}} + adp := &podadapter.AdapterInformer{PodInformer: informer} + return &realControl{ + adp: adp, + podReadinessControl: newPodReadinessControl(adp), + } } func GetPodLifecycleState(pod *v1.Pod) appspub.LifecycleStateType { return appspub.LifecycleStateType(pod.Labels[appspub.LifecycleStateKey]) } +func IsHookMarkPodNotReady(lifecycleHook *appspub.LifecycleHook) bool { + if lifecycleHook == nil { + return false + } + return lifecycleHook.MarkPodNotReady +} + +func IsLifecycleMarkPodNotReady(lifecycle *appspub.Lifecycle) bool { + if lifecycle == nil { + return false + } + return IsHookMarkPodNotReady(lifecycle.PreDelete) || IsHookMarkPodNotReady(lifecycle.InPlaceUpdate) +} + func SetPodLifecycle(state appspub.LifecycleStateType) func(*v1.Pod) { return func(pod *v1.Pod) { if pod.Labels == nil { @@ -71,7 +109,30 @@ func SetPodLifecycle(state appspub.LifecycleStateType) func(*v1.Pod) { } } -func (c *realControl) UpdatePodLifecycle(pod *v1.Pod, state appspub.LifecycleStateType) (updated bool, gotPod *v1.Pod, err error) { +func (c *realControl) executePodNotReadyPolicy(pod *v1.Pod, state appspub.LifecycleStateType) (err error) { + switch state { + case appspub.LifecycleStatePreparingDelete: + err = c.podReadinessControl.AddNotReadyKey(pod, getReadinessMessage(preparingDeleteHookKey)) + case appspub.LifecycleStatePreparingUpdate: + err = c.podReadinessControl.AddNotReadyKey(pod, getReadinessMessage(preparingUpdateHookKey)) + case appspub.LifecycleStateUpdated: + err = c.podReadinessControl.RemoveNotReadyKey(pod, getReadinessMessage(preparingUpdateHookKey)) + } + + if err != nil { + klog.Errorf("Failed to set pod(%v) Ready/NotReady at %s lifecycle state, error: %v", + client.ObjectKeyFromObject(pod), state, err) + } + return +} + +func (c *realControl) UpdatePodLifecycle(pod *v1.Pod, state appspub.LifecycleStateType, markPodNotReady bool) (updated bool, gotPod *v1.Pod, err error) { + if markPodNotReady { + if err = c.executePodNotReadyPolicy(pod, state); err != nil { + return false, nil, err + } + } + if GetPodLifecycleState(pod) == state { return false, pod, nil } @@ -99,6 +160,12 @@ func (c *realControl) UpdatePodLifecycleWithHandler(pod *v1.Pod, state appspub.L return false, pod, nil } + if inPlaceUpdateHandler.MarkPodNotReady { + if err = c.executePodNotReadyPolicy(pod, state); err != nil { + return false, nil, err + } + } + if GetPodLifecycleState(pod) == state { return false, pod, nil } @@ -173,3 +240,7 @@ func IsPodAllHooked(hook *appspub.LifecycleHook, pod *v1.Pod) bool { } return true } + +func getReadinessMessage(key string) podreadiness.Message { + return podreadiness.Message{UserAgent: "Lifecycle", Key: key} +} diff --git a/pkg/util/podreadiness/pod_readiness.go b/pkg/util/podreadiness/pod_readiness.go new file mode 100644 index 0000000000..4a73c663b2 --- /dev/null +++ b/pkg/util/podreadiness/pod_readiness.go @@ -0,0 +1,75 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podreadiness + +import ( + "sort" + + appspub "github.com/openkruise/kruise/apis/apps/pub" + "github.com/openkruise/kruise/pkg/util" + "github.com/openkruise/kruise/pkg/util/podadapter" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Interface interface { + AddNotReadyKey(pod *v1.Pod, msg Message) error + RemoveNotReadyKey(pod *v1.Pod, msg Message) error +} + +func New(c client.Client) Interface { + return &commonControl{adp: &podadapter.AdapterRuntimeClient{Client: c}} +} + +func NewForAdapter(adp podadapter.Adapter) Interface { + return &commonControl{adp: adp} +} + +type commonControl struct { + adp podadapter.Adapter +} + +func (c *commonControl) AddNotReadyKey(pod *v1.Pod, msg Message) error { + _, err := addNotReadyKey(c.adp, pod, msg, appspub.KruisePodReadyConditionType) + return err +} + +func (c *commonControl) RemoveNotReadyKey(pod *v1.Pod, msg Message) error { + _, err := removeNotReadyKey(c.adp, pod, msg, appspub.KruisePodReadyConditionType) + return err +} + +type Message struct { + UserAgent string `json:"userAgent"` + Key string `json:"key"` +} + +type messageList []Message + +func (c messageList) Len() int { return len(c) } +func (c messageList) Swap(i, j int) { c[i], c[j] = c[j], c[i] } +func (c messageList) Less(i, j int) bool { + if c[i].UserAgent == c[j].UserAgent { + return c[i].Key < c[j].Key + } + return c[i].UserAgent < c[j].UserAgent +} + +func (c messageList) dump() string { + sort.Sort(c) + return util.DumpJSON(c) +} diff --git a/pkg/util/podreadiness/pod_readiness_utils.go b/pkg/util/podreadiness/pod_readiness_utils.go index 925c8c56aa..cc402512de 100644 --- a/pkg/util/podreadiness/pod_readiness_utils.go +++ b/pkg/util/podreadiness/pod_readiness_utils.go @@ -17,33 +17,32 @@ limitations under the License. package podreadiness import ( - "context" "encoding/json" - "sort" appspub "github.com/openkruise/kruise/apis/apps/pub" - "github.com/openkruise/kruise/pkg/util" + "github.com/openkruise/kruise/pkg/util/podadapter" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" - "sigs.k8s.io/controller-runtime/pkg/client" ) -func AddNotReadyKey(c client.Client, pod *v1.Pod, msg Message) error { - if alreadyHasKey(pod, msg) { - return nil +func addNotReadyKey(adp podadapter.Adapter, pod *v1.Pod, msg Message, condType v1.PodConditionType) (bool, error) { + if alreadyHasKey(pod, msg, condType) { + return true, nil } - return retry.RetryOnConflict(retry.DefaultBackoff, func() error { - newPod := &v1.Pod{} - if err := c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, newPod); err != nil { + + readinessGateExists := false + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + newPod, err := adp.GetPod(pod.Namespace, pod.Name) + if err != nil { return err } - if !ContainsReadinessGate(pod) { + if !containsReadinessGate(newPod, condType) { return nil } - condition := GetReadinessCondition(newPod) + readinessGateExists = true + condition := getReadinessCondition(newPod, condType) if condition == nil { _, messages := addMessage("", msg) newPod.Status.Conditions = append(newPod.Status.Conditions, v1.PodCondition{ @@ -61,21 +60,24 @@ func AddNotReadyKey(c client.Client, pod *v1.Pod, msg Message) error { condition.LastTransitionTime = metav1.Now() } - return c.Status().Update(context.TODO(), newPod) + return adp.UpdatePodStatus(newPod) }) + return readinessGateExists, err } -func RemoveNotReadyKey(c client.Client, pod *v1.Pod, msg Message) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() error { - newPod := &v1.Pod{} - if err := c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, newPod); err != nil { +func removeNotReadyKey(adp podadapter.Adapter, pod *v1.Pod, msg Message, condType v1.PodConditionType) (bool, error) { + readinessGateExists := false + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + newPod, err := adp.GetPod(pod.Namespace, pod.Name) + if err != nil { return err } - if !ContainsReadinessGate(pod) { + if !containsReadinessGate(newPod, condType) { return nil } - condition := GetReadinessCondition(newPod) + readinessGateExists = true + condition := getReadinessCondition(newPod, condType) if condition == nil { return nil } @@ -89,29 +91,9 @@ func RemoveNotReadyKey(c client.Client, pod *v1.Pod, msg Message) error { condition.Message = messages.dump() condition.LastTransitionTime = metav1.Now() - return c.Status().Update(context.TODO(), newPod) + return adp.UpdatePodStatus(newPod) }) -} - -type Message struct { - UserAgent string `json:"userAgent"` - Key string `json:"key"` -} - -type messageList []Message - -func (c messageList) Len() int { return len(c) } -func (c messageList) Swap(i, j int) { c[i], c[j] = c[j], c[i] } -func (c messageList) Less(i, j int) bool { - if c[i].UserAgent == c[j].UserAgent { - return c[i].Key < c[j].Key - } - return c[i].UserAgent < c[j].UserAgent -} - -func (c messageList) dump() string { - sort.Sort(c) - return util.DumpJSON(c) + return readinessGateExists, err } func addMessage(base string, msg Message) (bool, messageList) { @@ -146,29 +128,37 @@ func removeMessage(base string, msg Message) (bool, messageList) { } func GetReadinessCondition(pod *v1.Pod) *v1.PodCondition { + return getReadinessCondition(pod, appspub.KruisePodReadyConditionType) +} + +func ContainsReadinessGate(pod *v1.Pod) bool { + return containsReadinessGate(pod, appspub.KruisePodReadyConditionType) +} + +func getReadinessCondition(pod *v1.Pod, condType v1.PodConditionType) *v1.PodCondition { if pod == nil { return nil } for i := range pod.Status.Conditions { c := &pod.Status.Conditions[i] - if c.Type == appspub.KruisePodReadyConditionType { + if c.Type == condType { return c } } return nil } -func ContainsReadinessGate(pod *v1.Pod) bool { +func containsReadinessGate(pod *v1.Pod, condType v1.PodConditionType) bool { for _, g := range pod.Spec.ReadinessGates { - if g.ConditionType == appspub.KruisePodReadyConditionType { + if g.ConditionType == condType { return true } } return false } -func alreadyHasKey(pod *v1.Pod, msg Message) bool { - condition := GetReadinessCondition(pod) +func alreadyHasKey(pod *v1.Pod, msg Message, condType v1.PodConditionType) bool { + condition := getReadinessCondition(pod, condType) if condition == nil { return false } diff --git a/pkg/util/podreadiness/pod_readiness_utils_test.go b/pkg/util/podreadiness/pod_readiness_utils_test.go index 5b196d2e44..b1d9751699 100644 --- a/pkg/util/podreadiness/pod_readiness_utils_test.go +++ b/pkg/util/podreadiness/pod_readiness_utils_test.go @@ -46,16 +46,20 @@ func TestPodReadiness(t *testing.T) { msg0 := Message{UserAgent: "ua1", Key: "foo"} msg1 := Message{UserAgent: "ua1", Key: "bar"} - if err := AddNotReadyKey(fakeClient, pod0, msg0); err != nil { + controller := New(fakeClient) + AddNotReadyKey := controller.AddNotReadyKey + RemoveNotReadyKey := controller.RemoveNotReadyKey + + if err := AddNotReadyKey(pod0, msg0); err != nil { t.Fatal(err) } - if err := AddNotReadyKey(fakeClient, pod0, msg1); err != nil { + if err := AddNotReadyKey(pod0, msg1); err != nil { t.Fatal(err) } - if err := AddNotReadyKey(fakeClient, pod1, msg0); err != nil { + if err := AddNotReadyKey(pod1, msg0); err != nil { t.Fatal(err) } - if err := AddNotReadyKey(fakeClient, pod1, msg1); err != nil { + if err := AddNotReadyKey(pod1, msg1); err != nil { t.Fatal(err) } @@ -63,7 +67,7 @@ func TestPodReadiness(t *testing.T) { if err := fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: pod0.Namespace, Name: pod0.Name}, newPod0); err != nil { t.Fatal(err) } - if !alreadyHasKey(newPod0, msg0) || !alreadyHasKey(newPod0, msg1) { + if !alreadyHasKey(newPod0, msg0, appspub.KruisePodReadyConditionType) || !alreadyHasKey(newPod0, msg1, appspub.KruisePodReadyConditionType) { t.Fatalf("expect already has key, but not") } condition := GetReadinessCondition(newPod0) @@ -75,24 +79,24 @@ func TestPodReadiness(t *testing.T) { if err := fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: pod1.Namespace, Name: pod1.Name}, newPod1); err != nil { t.Fatal(err) } - if alreadyHasKey(newPod1, msg0) || alreadyHasKey(newPod1, msg1) { + if alreadyHasKey(newPod1, msg0, appspub.KruisePodReadyConditionType) || alreadyHasKey(newPod1, msg1, appspub.KruisePodReadyConditionType) { t.Fatalf("expect not have key, but it does") } if condition = GetReadinessCondition(newPod1); condition != nil { t.Fatalf("expect condition nil, but exists: %v", condition) } - if err := RemoveNotReadyKey(fakeClient, newPod0, msg0); err != nil { + if err := RemoveNotReadyKey(newPod0, msg0); err != nil { t.Fatal(err) } newPod0 = &v1.Pod{} if err := fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: pod0.Namespace, Name: pod0.Name}, newPod0); err != nil { t.Fatal(err) } - if !alreadyHasKey(newPod0, msg1) { + if !alreadyHasKey(newPod0, msg1, appspub.KruisePodReadyConditionType) { t.Fatalf("expect already has key, but not") } - if alreadyHasKey(newPod0, msg0) { + if alreadyHasKey(newPod0, msg0, appspub.KruisePodReadyConditionType) { t.Fatalf("expect not have key, but it does") } condition = GetReadinessCondition(newPod0) @@ -100,14 +104,14 @@ func TestPodReadiness(t *testing.T) { t.Fatalf("expect condition false, but not") } - if err := RemoveNotReadyKey(fakeClient, newPod0, msg1); err != nil { + if err := RemoveNotReadyKey(newPod0, msg1); err != nil { t.Fatal(err) } newPod0 = &v1.Pod{} if err := fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: pod0.Namespace, Name: pod0.Name}, newPod0); err != nil { t.Fatal(err) } - if alreadyHasKey(newPod0, msg0) || alreadyHasKey(newPod0, msg1) { + if alreadyHasKey(newPod0, msg0, appspub.KruisePodReadyConditionType) || alreadyHasKey(newPod0, msg1, appspub.KruisePodReadyConditionType) { t.Fatalf("expect not have key, but it does") } condition = GetReadinessCondition(newPod0)