Merge branch 'master' into pvc_deletion_protection

CHANGELOG.md

@@ -1,5 +1,55 @@
 # Change Log
 
+## v1.4.0
+
+> Change log since v1.3.0
+
+### Upgrade Notice
+
+> No, really, you must read this before you upgrade
+
+- Enable following feature-gates by default: PreDownloadImageForInPlaceUpdate, ResourcesDeletionProtection, WorkloadSpread, PodUnavailableBudgetDeleteGate, InPlaceUpdateEnvFromMetadata,
+StatefulSetAutoDeletePVC, PodProbeMarkerGate. ([#1214](https://github.com/openkruise/kruise/pull/1214), [@zmberg](https://github.com/zmberg))
+- Change Kruise leader election from configmap to configmapsleases, this is a smooth upgrade with no disruption to OpenKruise service.  ([#1184](https://github.com/openkruise/kruise/pull/1184), [@YTGhost](https://github.com/YTGhost))
+
+### New Feature: JobSidecarTerminator
+
+In the Kubernetes world, it is challenging to use long-running sidecar containers for short-term job because there is no straightforward way to
+terminate the sidecar containers when the main container exits. For instance, when the main container in a Pod finishes its task and exits, it is expected that accompanying sidecars,
+such as a log collection sidecar, will also exit actively, so the Job Controller can accurately determine the completion status of the Pod.
+However most sidecar containers lack the ability to discovery the exit of main container.
+
+For this scenario OpenKruise provides the JobSidecarTerminator capability, which can terminate sidecar containers once the main containers exit.
+
+For more detail, please refer to its [proposal](https://github.com/openkruise/kruise/blob/master/docs/proposals/20230130-job-sidecar-terminator.md).
+
+### Advanced Workloads
+- Optimized CloneSet Event handler to reduce unnecessary reconciliation. The feature is off by default and controlled by CloneSetEventHandlerOptimization feature-gate. ([#1219](https://github.com/openkruise/kruise/pull/1219), [@veophi](https://github.com/veophi))
+- Avoid pod hang in PreparingUpdate state when rollback before update hook. ([#1157](https://github.com/openkruise/kruise/pull/1157), [@shiyan2016](https://github.com/shiyan2016))
+- Fix cloneSet update blocking when spec.scaleStrategy.maxUnavailable is not empty. ([#1136](https://github.com/openkruise/kruise/pull/1136), [@ivanszl](https://github.com/ivanszl))
+- Add 'disablePVCReuse' field to enable recreation of PVCs when rebuilding pods, which can avoid Pod creation failure due to Node exceptions. ([#1113](https://github.com/openkruise/kruise/pull/1113), [@willise](https://github.com/willise))
+- CloneSet 'partition' field support float percent to improve precision. ([#1124](https://github.com/openkruise/kruise/pull/1124), [@shiyan2016](https://github.com/shiyan2016))
+- Add PreNormal Lifecycle Hook for CloneSet. ([#1071](https://github.com/openkruise/kruise/pull/1071), [@veophi](https://github.com/veophi))
+- Allow to mutate PVCTemplate of Advanced StatefulSet & CloneSet. Note, Only works for new Pods, not for existing Pods. ([#1118](https://github.com/openkruise/kruise/pull/1118), [@veophi](https://github.com/veophi))
+- Make ephemeralJob compatible with k8s version 1.20 & 1.21. ([#1127](https://github.com/openkruise/kruise/pull/1127), [@veophi](https://github.com/veophi))
+- UnitedDeployment support advanced StatefulSet 'persistentVolumeClaimRetentionPolicy' field. ([#1110](https://github.com/openkruise/kruise/pull/1110), [@yuexian1234](https://github.com/yuexian1234))
+
+### ContainerRecreateRequest
+- Add 'forceRecreate' field to ensure the immediate recreation of the container even if the container is starting at that point. ([#1182](https://github.com/openkruise/kruise/pull/1182), [@BH4AWS](https://github.com/BH4AWS))
+
+### ImagePullJob
+- Support attach metadata in PullImage CRI interface during ImagePullJob. ([#1190](https://github.com/openkruise/kruise/pull/1190), [@diannaowa](https://github.com/diannaowa))
+
+### SidecarSet
+- SidecarSet support namespace selector. ([#1178](https://github.com/openkruise/kruise/pull/1178), [@zmberg](https://github.com/zmberg))
+
+### Others
+- Simplify some code, mainly comparison and variable declaration. ([#1209](https://github.com/openkruise/kruise/pull/1209), [@hezhizhen](https://github.com/hezhizhen))
+- Update k8s registry references from k8s.gcr.io to registry.k8s.io. ([#1208](https://github.com/openkruise/kruise/pull/1208), [@asa3311](https://github.com/asa3311))
+- Fix config/samples/apps_v1alpha1_uniteddeployment.yaml invalid image. ([#1198](https://github.com/openkruise/kruise/pull/1198), [@chengleqi](https://github.com/chengleqi))
+- Change kruise base image to alpine. ([#1166](https://github.com/openkruise/kruise/pull/1166), [@fengshunli](https://github.com/fengshunli))
+- PersistentPodState support custom workload (like statefulSet). ([#1063](https://github.com/openkruise/kruise/pull/1063), [@baxiaoshi](https://github.com/baxiaoshi))
+
 ## v1.3.0
 
 > Change log since v1.2.0

apis/apps/defaults/v1alpha1.go

@@ -192,6 +192,11 @@ func SetDefaultsBroadcastJob(obj *v1alpha1.BroadcastJob, injectTemplateDefaults
 	if obj.Spec.FailurePolicy.Type == "" {
 		obj.Spec.FailurePolicy.Type = v1alpha1.FailurePolicyTypeFailFast
 	}
+
+	// Default to 'OnFailure' if no restartPolicy is specified
+	if obj.Spec.Template.Spec.RestartPolicy == "" {
+		obj.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyOnFailure
+	}
 }
 
 // SetDefaults_UnitedDeployment set default values for UnitedDeployment.

pkg/controller/broadcastjob/broadcastjob_controller.go

@@ -524,14 +524,9 @@ func getNodesToRunPod(nodes *corev1.NodeList, job *appsv1alpha1.BroadcastJob,
 		// there's pod existing on the node
 		if pod, ok := existingNodeToPodMap[node.Name]; ok {
 			canFit, err = checkNodeFitness(pod, &node)
-			if err != nil {
-				klog.Errorf("pod %s failed to checkNodeFitness for node %s, %v", pod.Name, node.Name, err)
-				continue
-			}
-			if !canFit {
-				if pod.DeletionTimestamp == nil {
-					podsToDelete = append(podsToDelete, pod)
-				}
+			if !canFit && pod.DeletionTimestamp == nil {
+				klog.Infof("Pod %s does not fit on node %s due to %v", pod.Name, node.Name, err)
+				podsToDelete = append(podsToDelete, pod)
 				continue
 			}
 			desiredNodes[node.Name] = pod
@@ -540,12 +535,8 @@ func getNodesToRunPod(nodes *corev1.NodeList, job *appsv1alpha1.BroadcastJob,
 			// considering nodeName, label affinity and taints
 			mockPod := NewMockPod(job, node.Name)
 			canFit, err = checkNodeFitness(mockPod, &node)
-			if err != nil {
-				klog.Errorf("failed to checkNodeFitness for node %s, %v", node.Name, err)
-				continue
-			}
 			if !canFit {
-				klog.Infof("Pod does not fit on node %s", node.Name)
+				klog.Infof("Pod does not fit on node %s due to %v", node.Name, err)
 				continue
 			}
 			restNodesToRunPod = append(restNodesToRunPod, &nodes.Items[i])

pkg/controller/broadcastjob/broadcastjob_event_handler.go

@@ -108,12 +108,8 @@ func (p *enqueueBroadcastJobForNode) addNode(q workqueue.RateLimitingInterface,
 	for _, bcj := range jobList.Items {
 		mockPod := NewMockPod(&bcj, node.Name)
 		canFit, err := checkNodeFitness(mockPod, node)
-		if err != nil {
-			klog.Errorf("failed to checkNodeFitness for job %s/%s, on node %s, %v", bcj.Namespace, bcj.Name, node.Name, err)
-			continue
-		}
 		if !canFit {
-			klog.Infof("Job %s/%s does not fit on node %s", bcj.Namespace, bcj.Name, node.Name)
+			klog.Infof("Job %s/%s does not fit on node %s due to %v", bcj.Namespace, bcj.Name, node.Name, err)
 			continue
 		}
 
@@ -138,17 +134,8 @@ func (p *enqueueBroadcastJobForNode) updateNode(q workqueue.RateLimitingInterfac
 	}
 	for _, bcj := range jobList.Items {
 		mockPod := NewMockPod(&bcj, oldNode.Name)
-		canOldNodeFit, err := checkNodeFitness(mockPod, oldNode)
-		if err != nil {
-			klog.Errorf("failed to checkNodeFitness for job %s/%s, on old node %s, %v", bcj.Namespace, bcj.Name, oldNode.Name, err)
-			continue
-		}
-
-		canCurNodeFit, err := checkNodeFitness(mockPod, curNode)
-		if err != nil {
-			klog.Errorf("failed to checkNodeFitness for job %s/%s, on cur node %s, %v", bcj.Namespace, bcj.Name, curNode.Name, err)
-			continue
-		}
+		canOldNodeFit, _ := checkNodeFitness(mockPod, oldNode)
+		canCurNodeFit, _ := checkNodeFitness(mockPod, curNode)
 
 		if canOldNodeFit != canCurNodeFit {
 			// enqueue the broadcast job for matching node

pkg/daemon/criruntime/factory.go

@@ -96,13 +96,7 @@ func NewFactory(varRunPath string, accountManager daemonutil.ImagePullAccountMan
 				klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err)
 				continue
 			}
-		case ContainerRuntimePouch:
-			imageService, err = runtimeimage.NewPouchImageService(cfg.runtimeURI, accountManager)
-			if err != nil {
-				klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err)
-				continue
-			}
-		case ContainerRuntimeContainerd, ContainerRuntimeCommonCRI:
+		case ContainerRuntimeContainerd, ContainerRuntimeCommonCRI, ContainerRuntimePouch:
 			addr, _, err := kubeletutil.GetAddressAndDialer(cfg.runtimeRemoteURI)
 			if err != nil {
 				klog.Warningf("Failed to get address for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err)

pkg/webhook/pod/mutating/sidecarset.go

@@ -362,15 +362,27 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
 		if !isUpdated {
 			for i := range sidecarSet.Spec.InitContainers {
 				initContainer := &sidecarSet.Spec.InitContainers[i]
-				//add "IS_INJECTED" env in initContainer's envs
-				initContainer.Env = append(initContainer.Env, corev1.EnvVar{Name: sidecarcontrol.SidecarEnvKey, Value: "true"})
+				// volumeMounts that injected into sidecar container
+				// when volumeMounts SubPathExpr contains expansions, then need copy container EnvVars(injectEnvs)
+				injectedMounts, injectedEnvs := sidecarcontrol.GetInjectedVolumeMountsAndEnvs(control, initContainer, pod)
+				// get injected env & mounts explicitly so that can be compared with old ones in pod
 				transferEnvs := sidecarcontrol.GetSidecarTransferEnvs(initContainer, pod)
-				initContainer.Env = append(initContainer.Env, transferEnvs...)
-				sidecarInitContainers = append(sidecarInitContainers, initContainer)
+				// append volumeMounts SubPathExpr environments
+				transferEnvs = util.MergeEnvVar(transferEnvs, injectedEnvs)
+				klog.Infof("try to inject initContainer sidecar %v@%v/%v, with injected envs: %v, volumeMounts: %v",
+					initContainer.Name, pod.Namespace, pod.Name, transferEnvs, injectedMounts)
 				// insert volumes that initContainers used
 				for _, mount := range initContainer.VolumeMounts {
 					volumesInSidecars = append(volumesInSidecars, *volumesMap[mount.Name])
 				}
+				// merge VolumeMounts from sidecar.VolumeMounts and shared VolumeMounts
+				initContainer.VolumeMounts = util.MergeVolumeMounts(initContainer.VolumeMounts, injectedMounts)
+				// add "IS_INJECTED" env in initContainer's envs
+				initContainer.Env = append(initContainer.Env, corev1.EnvVar{Name: sidecarcontrol.SidecarEnvKey, Value: "true"})
+				// merged Env from sidecar.Env and transfer envs
+				initContainer.Env = util.MergeEnvVar(initContainer.Env, transferEnvs)
+
+				sidecarInitContainers = append(sidecarInitContainers, initContainer)
 			}
 			//process imagePullSecrets
 			sidecarSecrets = append(sidecarSecrets, sidecarSet.Spec.ImagePullSecrets...)
@@ -389,7 +401,7 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
 			transferEnvs := sidecarcontrol.GetSidecarTransferEnvs(sidecarContainer, pod)
 			// append volumeMounts SubPathExpr environments
 			transferEnvs = util.MergeEnvVar(transferEnvs, injectedEnvs)
-			klog.Infof("try to inject sidecar %v@%v/%v, with injected envs: %v, volumeMounts: %v",
+			klog.Infof("try to inject Container sidecar %v@%v/%v, with injected envs: %v, volumeMounts: %v",
 				sidecarContainer.Name, pod.Namespace, pod.Name, transferEnvs, injectedMounts)
 			//when update pod object
 			if isUpdated {

pkg/webhook/pod/mutating/sidecarset_test.go

@@ -238,6 +238,32 @@ var (
 					"app": "suxing-test",
 				},
 			},
+			InitContainers: []appsv1alpha1.SidecarContainer{
+				{
+					Container: corev1.Container{
+						Name:  "dns-e",
+						Image: "dns-e-image:1.0",
+						VolumeMounts: []corev1.VolumeMount{
+							{
+								Name:      "volume-3",
+								MountPath: "/g/h/i",
+							},
+							{
+								Name:      "volume-4",
+								MountPath: "/j/k/l",
+							},
+							{
+								Name:      "volume-staragent",
+								MountPath: "/staragent",
+							},
+						},
+					},
+					PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
+					ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
+						Type: appsv1alpha1.ShareVolumePolicyEnabled,
+					},
+				},
+			},
 			Containers: []appsv1alpha1.SidecarContainer{
 				{
 					Container: corev1.Container{
@@ -288,6 +314,8 @@ var (
 				{Name: "volume-1"},
 				{Name: "volume-2"},
 				{Name: "volume-staragent"},
+				{Name: "volume-3"},
+				{Name: "volume-4"},
 			},
 		},
 	}
@@ -730,11 +758,12 @@ func testPodVolumeMountsAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarS
 	// /a/b、/e/f
 	podIn := podWithStaragent.DeepCopy()
 	cases := []struct {
-		name               string
-		getPod             func() *corev1.Pod
-		getSidecarSets     func() *appsv1alpha1.SidecarSet
-		exceptVolumeMounts []string
-		exceptEnvs         []string
+		name                   string
+		getPod                 func() *corev1.Pod
+		getSidecarSets         func() *appsv1alpha1.SidecarSet
+		exceptInitVolumeMounts []string
+		exceptVolumeMounts     []string
+		exceptEnvs             []string
 	}{
 		{
 			name: "append normal volumeMounts",
@@ -744,7 +773,8 @@ func testPodVolumeMountsAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarS
 			getSidecarSets: func() *appsv1alpha1.SidecarSet {
 				return sidecarSetIn.DeepCopy()
 			},
-			exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent"},
+			exceptInitVolumeMounts: []string{"/a/b", "/e/f", "/g/h/i", "/j/k/l", "/staragent"},
+			exceptVolumeMounts:     []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent"},
 		},
 		{
 			name: "append volumeMounts SubPathExpr, volumes with expanded subpath",
@@ -768,8 +798,9 @@ func testPodVolumeMountsAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarS
 			getSidecarSets: func() *appsv1alpha1.SidecarSet {
 				return sidecarSetIn.DeepCopy()
 			},
-			exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"},
-			exceptEnvs:         []string{"POD_NAME", "OD_NAME"},
+			exceptInitVolumeMounts: []string{"/a/b", "/e/f", "/g/h/i", "/j/k/l", "/staragent", "/e/expansion"},
+			exceptVolumeMounts:     []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"},
+			exceptEnvs:             []string{"POD_NAME", "OD_NAME"},
 		},
 		{
 			name: "append volumeMounts SubPathExpr, subpath with no expansion",
@@ -785,7 +816,8 @@ func testPodVolumeMountsAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarS
 			getSidecarSets: func() *appsv1alpha1.SidecarSet {
 				return sidecarSetIn.DeepCopy()
 			},
-			exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"},
+			exceptInitVolumeMounts: []string{"/a/b", "/e/f", "/g/h/i", "/j/k/l", "/staragent", "/e/expansion"},
+			exceptVolumeMounts:     []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"},
 		},
 		{
 			name: "append volumeMounts SubPathExpr, volumes expanded with empty subpath",
@@ -801,7 +833,8 @@ func testPodVolumeMountsAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarS
 			getSidecarSets: func() *appsv1alpha1.SidecarSet {
 				return sidecarSetIn.DeepCopy()
 			},
-			exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"},
+			exceptInitVolumeMounts: []string{"/a/b", "/e/f", "/g/h/i", "/j/k/l", "/staragent", "/e/expansion"},
+			exceptVolumeMounts:     []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"},
 		},
 	}
 
@@ -818,15 +851,27 @@ func testPodVolumeMountsAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarS
 				t.Fatalf("inject sidecar into pod failed, err: %v", err)
 			}
 
+			for _, mount := range cs.exceptInitVolumeMounts {
+				if util.GetContainerVolumeMount(&podOut.Spec.InitContainers[0], mount) == nil {
+					t.Fatalf("expect volume mounts in InitContainer %s but got nil", mount)
+				}
+			}
+
+			for _, env := range cs.exceptEnvs {
+				if util.GetContainerEnvVar(&podOut.Spec.InitContainers[0], env) == nil {
+					t.Fatalf("expect env in InitContainer %s but got nil", env)
+				}
+			}
+
 			for _, mount := range cs.exceptVolumeMounts {
 				if util.GetContainerVolumeMount(&podOut.Spec.Containers[1], mount) == nil {
-					t.Fatalf("expect volume mounts %s but got nil", mount)
+					t.Fatalf("expect volume mounts in Container %s but got nil", mount)
 				}
 			}
 
 			for _, env := range cs.exceptEnvs {
 				if util.GetContainerEnvVar(&podOut.Spec.Containers[1], env) == nil {
-					t.Fatalf("expect env %s but got nil", env)
+					t.Fatalf("expect env in Container %s but got nil", env)
 				}
 			}
 		})