Skip to content

Commit

Permalink
sync podCondition when probe message of NodePodProbe changed
Browse files Browse the repository at this point in the history 
Signed-off-by: ChrisLiu <[email protected]>
  • Loading branch information
@chrisliu1995
chrisliu1995 committed Jan 5, 2024
1 parent fa9a9a0 commit b41a89a
Show file tree
Hide file tree
Showing 5 changed files with 322 additions and 2 deletions.
2 changes: 1 addition & 1 deletion pkg/controller/nodepodprobe/node_pod_probe_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ func (r *ReconcileNodePodProbe) updatePodProbeStatus(pod *corev1.Pod, status app
oldStatus := podClone.Status.DeepCopy()
for i := range probeConditions {
condition := probeConditions[i]
util.SetPodCondition(podClone, condition)
util.SetPodConditionIfMsgChanged(podClone, condition)
}
oldMetadata := podClone.ObjectMeta.DeepCopy()
if podClone.Annotations == nil {
Expand Down
12 changes: 12 additions & 0 deletions pkg/util/pods.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,18 @@ func SetPodCondition(pod *v1.Pod, condition v1.PodCondition) {
pod.Status.Conditions = append(pod.Status.Conditions, condition)
}

func SetPodConditionIfMsgChanged(pod *v1.Pod, condition v1.PodCondition) {
for i, c := range pod.Status.Conditions {
if c.Type == condition.Type {
if c.Status != condition.Status || c.Message != condition.Message {
pod.Status.Conditions[i] = condition
}
return
}
}
pod.Status.Conditions = append(pod.Status.Conditions, condition)
}

func SetPodReadyCondition(pod *v1.Pod) {
podReady := GetCondition(pod, v1.PodReady)
if podReady == nil {
Expand Down
124 changes: 124 additions & 0 deletions pkg/util/pods_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package util

import (
"reflect"
"testing"

v1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -102,3 +103,126 @@ func TestMergeVolumes(t *testing.T) {
}
}
}

func TestSetPodConditionIfMsgChanged(t *testing.T) {
tests := []struct {
pod *v1.Pod
condition v1.PodCondition
conditions []v1.PodCondition
}{
// case 0: existed condition status changed
{
pod: &v1.Pod{
Status: v1.PodStatus{
Conditions: []v1.PodCondition{
{
Type: "type-0",
Status: v1.ConditionTrue,
Message: "Msg-0",
},
{
Type: "type-1",
Status: v1.ConditionFalse,
},
},
},
},
condition: v1.PodCondition{
Type: "type-0",
Status: v1.ConditionFalse,
},
conditions: []v1.PodCondition{
{
Type: "type-0",
Status: v1.ConditionFalse,
},
{
Type: "type-1",
Status: v1.ConditionFalse,
},
},
},

// case 1: add a new condition
{
pod: &v1.Pod{
Status: v1.PodStatus{
Conditions: []v1.PodCondition{
{
Type: "type-0",
Status: v1.ConditionTrue,
Message: "Msg-0",
},
{
Type: "type-1",
Status: v1.ConditionFalse,
},
},
},
},
condition: v1.PodCondition{
Type: "type-2",
Status: v1.ConditionFalse,
},
conditions: []v1.PodCondition{
{
Type: "type-0",
Status: v1.ConditionTrue,
Message: "Msg-0",
},
{
Type: "type-1",
Status: v1.ConditionFalse,
},
{
Type: "type-2",
Status: v1.ConditionFalse,
},
},
},

// case 2: existed condition status not changed, but message changed
{
pod: &v1.Pod{
Status: v1.PodStatus{
Conditions: []v1.PodCondition{
{
Type: "type-0",
Status: v1.ConditionTrue,
Message: "Msg-0",
},
{
Type: "type-1",
Status: v1.ConditionFalse,
},
},
},
},
condition: v1.PodCondition{
Type: "type-0",
Status: v1.ConditionTrue,
Message: "Msg-Changed",
},
conditions: []v1.PodCondition{
{
Type: "type-0",
Status: v1.ConditionTrue,
Message: "Msg-Changed",
},
{
Type: "type-1",
Status: v1.ConditionFalse,
},
},
},
}

for i, test := range tests {
expect := test.conditions
SetPodConditionIfMsgChanged(test.pod, test.condition)
actual := test.pod.Status.Conditions
if !reflect.DeepEqual(expect, actual) {
t.Fatalf("case %d: expect Conditions(%s), but get %s", i, expect, actual)
}
}
}
113 changes: 113 additions & 0 deletions test/e2e/apps/podprobemarker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package apps
import (
"context"
"fmt"
"k8s.io/apimachinery/pkg/util/wait"
"strings"
"time"

"github.com/onsi/ginkgo"
Expand Down Expand Up @@ -273,5 +275,116 @@ var _ = SIGDescribe("PodProbeMarker", func() {
gomega.Expect(npp.Spec.PodProbes).To(gomega.HaveLen(0))
}
})

ginkgo.It("pod probe marker test2", func() {
nodeList, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
nodeLen := len(nodeList.Items)
if nodeLen == 0 {
ginkgo.By("pod probe markers list nodeList is zero")
return
}
nppList, err := kc.AppsV1alpha1().NodePodProbes().List(context.TODO(), metav1.ListOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(nppList.Items).To(gomega.HaveLen(nodeLen))

// create statefulset
sts := tester.NewStatefulSetWithProbeImg(ns, randStr)
// For heterogeneous scenario like edge cluster, I want to deploy a Pod for each Node to verify that the functionality works
sts.Spec.Template.Spec.TopologySpreadConstraints = []v1.TopologySpreadConstraint{
{
LabelSelector: sts.Spec.Selector,
MaxSkew: 1,
TopologyKey: "kubernetes.io/hostname",
WhenUnsatisfiable: v1.ScheduleAnyway,
},
}
sts.Spec.Replicas = utilpointer.Int32Ptr(int32(nodeLen))
ginkgo.By(fmt.Sprintf("Create statefulset(%s/%s)", sts.Namespace, sts.Name))
tester.CreateStatefulSet(sts)

// create pod probe marker
ppm := tester.NewPodProbeMarkerWithProbeImg(ns, randStr)
_, err = kc.AppsV1alpha1().PodProbeMarkers(ns).Create(context.TODO(), &ppm, metav1.CreateOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())

// Firstly, probe.sh return false
err = wait.PollImmediate(1*time.Second, 10*time.Second,
func() (done bool, err error) {
pods, err := tester.ListActivePods(ns)
if err != nil {
return false, err
}
if len(pods) != nodeLen {
return false, nil
}

for _, pod := range pods {
condition := util.GetCondition(pod, "game.kruise.io/healthy")
if condition == nil {
return false, nil
}
if condition.Status != v1.ConditionFalse {
return false, nil
}
}
return true, nil
})
gomega.Expect(err).NotTo(gomega.HaveOccurred())

// Secondly, probe.sh return true & msg is 'data'
err = wait.PollImmediate(1*time.Second, 10*time.Second,
func() (done bool, err error) {
pods, err := tester.ListActivePods(ns)
if err != nil {
return false, err
}
if len(pods) != nodeLen {
return false, fmt.Errorf("the num of pods(%d) is not same as num of nodes(%d)", len(pods), nodeLen)
}

for _, pod := range pods {
condition := util.GetCondition(pod, "game.kruise.io/healthy")
if condition == nil {
return false, nil
}
if condition.Status != v1.ConditionTrue {
return false, nil
}
if !strings.Contains(condition.Message, "data") {
return false, nil
}
}
return true, nil
})
gomega.Expect(err).NotTo(gomega.HaveOccurred())

// Thirdly, probe.sh return true & msg is 'gate'
err = wait.PollImmediate(1*time.Second, 10*time.Second,
func() (done bool, err error) {
pods, err := tester.ListActivePods(ns)
if err != nil {
return false, err
}
if len(pods) != nodeLen {
return false, fmt.Errorf("the num of pods(%d) is not same as num of nodes(%d)", len(pods), nodeLen)
}

for _, pod := range pods {
condition := util.GetCondition(pod, "game.kruise.io/healthy")
if condition == nil {
return false, nil
}
if condition.Status != v1.ConditionTrue {
return false, nil
}
if !strings.Contains(condition.Message, "gate") {
return false, nil
}
}
return true, nil
})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
})
})
})
73 changes: 72 additions & 1 deletion test/e2e/framework/pod_probe_marker_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,77 @@ func (s *PodProbeMarkerTester) NewBaseStatefulSet(namespace, randStr string) *ap
}
}

func (s *PodProbeMarkerTester) NewPodProbeMarkerWithProbeImg(ns, randStr string) appsv1alpha1.PodProbeMarker {
return appsv1alpha1.PodProbeMarker{
ObjectMeta: metav1.ObjectMeta{
Name: "ppm-minecraft",
Namespace: ns,
},
Spec: appsv1alpha1.PodProbeMarkerSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": fmt.Sprintf("probe-%s", randStr),
},
},
Probes: []appsv1alpha1.PodContainerProbe{
{
Name: "healthy",
ContainerName: "minecraft",
Probe: appsv1alpha1.ContainerProbeSpec{
Probe: corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
Exec: &corev1.ExecAction{
Command: []string{"bash", "./probe.sh"},
},
},
},
},
PodConditionType: "game.kruise.io/healthy",
},
},
},
}
}

func (s *PodProbeMarkerTester) NewStatefulSetWithProbeImg(namespace, randStr string) *appsv1beta1.StatefulSet {
return &appsv1beta1.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps.kruise.io/v1beta1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "stateful-test",
Namespace: namespace,
},
Spec: appsv1beta1.StatefulSetSpec{
PodManagementPolicy: apps.ParallelPodManagement,
ServiceName: "fake-service",
Replicas: utilpointer.Int32Ptr(2),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": fmt.Sprintf("probe-%s", randStr),
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": fmt.Sprintf("probe-%s", randStr),
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "minecraft",
Image: "openkruise/minecraft-demo:probe-v0",
ImagePullPolicy: corev1.PullIfNotPresent,
},
},
},
},
},
}
}

func (s *PodProbeMarkerTester) CreateStatefulSet(sts *appsv1beta1.StatefulSet) {
Logf("create sts(%s/%s)", sts.Namespace, sts.Name)
_, err := s.kc.AppsV1beta1().StatefulSets(sts.Namespace).Create(context.TODO(), sts, metav1.CreateOptions{})
Expand All @@ -185,7 +256,7 @@ func (s *PodProbeMarkerTester) CreateStatefulSet(sts *appsv1beta1.StatefulSet) {
}

func (s *PodProbeMarkerTester) WaitForStatefulSetRunning(sts *appsv1beta1.StatefulSet) {
pollErr := wait.PollImmediate(time.Second, time.Minute,
pollErr := wait.PollImmediate(time.Second, 2*time.Minute,
func() (bool, error) {
inner, err := s.kc.AppsV1beta1().StatefulSets(sts.Namespace).Get(context.TODO(), sts.Name, metav1.GetOptions{})
if err != nil {
Expand Down

0 comments on commit b41a89a

Please sign in to comment.