diff --git a/.gitignore b/.gitignore index 10b1c97c09d..7275935ef78 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ tests/images/e2e/tidb-cluster/ tests/images/e2e/tidb-backup/ tests/images/e2e/tidb-operator/ tests/images/e2e/manifests/ +manifests/crd-verify.yaml *.tar tmp/ data/ @@ -30,8 +31,6 @@ coverage.out vendor tests/e2e/e2e.test .orig -tkc - apiserver.local.config/ manifests/crd-verify.yaml /tkctl diff --git a/charts/tidb-cluster/templates/tidb-cluster.yaml b/charts/tidb-cluster/templates/tidb-cluster.yaml index e5734a59a1e..806af3d4008 100644 --- a/charts/tidb-cluster/templates/tidb-cluster.yaml +++ b/charts/tidb-cluster/templates/tidb-cluster.yaml @@ -19,6 +19,7 @@ metadata: {{- end }} spec: pvReclaimPolicy: {{ .Values.pvReclaimPolicy }} + enablePVReclaim: {{ .Values.enablePVReclaim }} timezone: {{ .Values.timezone | default "UTC" }} enableTLSCluster: {{ .Values.enableTLSCluster | default false }} enableTLSClient: {{ .Values.enableTLSClient | default false }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index f8923865b2b..bccfd057239 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -22,11 +22,15 @@ schedulerName: tidb-scheduler # timezone is the default system timzone for TiDB timezone: UTC -# reclaim policy of a PV, default: Retain. +# reclaim policy of PV, default: Retain. # you must set it to Retain to ensure data safety in production environment. # https://pingcap.com/docs/v3.0/tidb-in-kubernetes/reference/configuration/local-pv/#data-security pvReclaimPolicy: Retain +# Whether enable PV reclaim. +# When enabling, scale in pd or tikv will trigger PV reclaim automatically +enablePVReclaim: false + # services is the service list to expose, default is ClusterIP # can be ClusterIP | NodePort | LoadBalancer services: diff --git a/manifests/crd.yaml b/manifests/crd.yaml index a35cfeabb26..19b85632d23 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -74,6 +74,8 @@ spec: description: TidbClusterSpec describes the attributes that a user creates on a tidb cluster properties: + enablePVReclaim: + type: boolean enableTLSCluster: description: Enable TLS connection between TiDB server compoments type: boolean diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 47ff7d7217a..14160fb53de 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -1119,6 +1119,12 @@ func schema_pkg_apis_pingcap_v1alpha1_TidbClusterSpec(ref common.ReferenceCallba Format: "", }, }, + "enablePVReclaim": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, "timezone": { SchemaProps: spec.SchemaProps{ Type: []string{"string"}, diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index a3294bf8fa3..ac87c057a3f 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -99,6 +99,7 @@ type TidbClusterSpec struct { // Services list non-headless services type used in TidbCluster Services []Service `json:"services,omitempty"` PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"` + EnablePVReclaim bool `json:"enablePVReclaim,omitempty"` Timezone string `json:"timezone,omitempty"` // Enable TLS connection between TiDB server compoments EnableTLSCluster bool `json:"enableTLSCluster,omitempty"` diff --git a/pkg/controller/pv_control.go b/pkg/controller/pv_control.go index 3d7f2df1ced..6a705d2bdb9 100644 --- a/pkg/controller/pv_control.go +++ b/pkg/controller/pv_control.go @@ -200,7 +200,7 @@ func (fpc *FakePVControl) PatchPVReclaimPolicy(_ *v1alpha1.TidbCluster, pv *core defer fpc.updatePVTracker.Inc() if fpc.updatePVTracker.ErrorReady() { defer fpc.updatePVTracker.Reset() - return fpc.updatePVTracker.err + return fpc.updatePVTracker.GetError() } pv.Spec.PersistentVolumeReclaimPolicy = reclaimPolicy @@ -212,7 +212,7 @@ func (fpc *FakePVControl) UpdateMetaInfo(tc *v1alpha1.TidbCluster, pv *corev1.Pe defer fpc.updatePVTracker.Inc() if fpc.updatePVTracker.ErrorReady() { defer fpc.updatePVTracker.Reset() - return nil, fpc.updatePVTracker.err + return nil, fpc.updatePVTracker.GetError() } ns := tc.GetNamespace() pvcName := pv.Spec.ClaimRef.Name diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index 5a6b6f1950b..d2a063f4a48 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -170,9 +170,12 @@ func NewController( kubeCli, ), mm.NewRealPVCCleaner( + kubeCli, podInformer.Lister(), pvcControl, pvcInformer.Lister(), + pvInformer.Lister(), + pvControl, ), recorder, ), diff --git a/pkg/manager/member/pvc_cleaner.go b/pkg/manager/member/pvc_cleaner.go index 8b153c451ff..1a2b4532662 100644 --- a/pkg/manager/member/pvc_cleaner.go +++ b/pkg/manager/member/pvc_cleaner.go @@ -21,6 +21,8 @@ import ( "github.com/pingcap/tidb-operator/pkg/label" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" corelisters "k8s.io/client-go/listers/core/v1" glog "k8s.io/klog" ) @@ -31,7 +33,14 @@ const ( skipReasonPVCCleanerPVCNotHasLock = "pvc cleaner: pvc not has schedule lock" skipReasonPVCCleanerPodWaitingForScheduling = "pvc cleaner: waiting for pod scheduling" skipReasonPVCCleanerPodNotFound = "pvc cleaner: the corresponding pod of pvc has not been found" - skipReasonPVCCleanerWaitingForPVCSync = "pvc cleaner: waiting for pvc's meta info to be synced" + skipReasonPVCCleanerPVCNotBound = "pvc cleaner: the pvc is not bound" + skipReasonPVCCleanerPVCNotHasPodNameAnn = "pvc cleaner: pvc has no pod name annotation" + skipReasonPVCCleanerIsNotDeferDeletePVC = "pvc cleaner: pvc has not been marked as defer delete pvc" + skipReasonPVCCleanerPVCeferencedByPod = "pvc cleaner: pvc is still referenced by a pod" + skipReasonPVCCleanerNotFoundPV = "pvc cleaner: not found pv bound to pvc" + skipReasonPVCCleanerPVCHasBeenDeleted = "pvc cleaner: pvc has been deleted" + skipReasonPVCCleanerPVCNotFound = "pvc cleaner: not found pvc from apiserver" + skipReasonPVCCleanerPVCChanged = "pvc cleaner: pvc changed before deletion" ) // PVCCleaner implements the logic for cleaning the pvc related resource @@ -40,37 +49,161 @@ type PVCCleanerInterface interface { } type realPVCCleaner struct { + kubeCli kubernetes.Interface podLister corelisters.PodLister pvcControl controller.PVCControlInterface pvcLister corelisters.PersistentVolumeClaimLister + pvLister corelisters.PersistentVolumeLister + pvControl controller.PVControlInterface } // NewRealPVCCleaner returns a realPVCCleaner func NewRealPVCCleaner( + kubeCli kubernetes.Interface, podLister corelisters.PodLister, pvcControl controller.PVCControlInterface, - pvcLister corelisters.PersistentVolumeClaimLister) PVCCleanerInterface { + pvcLister corelisters.PersistentVolumeClaimLister, + pvLister corelisters.PersistentVolumeLister, + pvControl controller.PVControlInterface) PVCCleanerInterface { return &realPVCCleaner{ + kubeCli, podLister, pvcControl, pvcLister, + pvLister, + pvControl, } } func (rpc *realPVCCleaner) Clean(tc *v1alpha1.TidbCluster) (map[string]string, error) { + if skipReason, err := rpc.cleanScheduleLock(tc); err != nil { + return skipReason, err + } + + if !tc.Spec.EnablePVReclaim { + // disable PV reclaim, return directly. + return nil, nil + } + return rpc.reclaimPV(tc) +} + +func (rpc *realPVCCleaner) reclaimPV(tc *v1alpha1.TidbCluster) (map[string]string, error) { ns := tc.GetNamespace() tcName := tc.GetName() + // for unit test skipReason := map[string]string{} - selector, err := label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).Selector() + pvcs, err := rpc.listAllPVCs(tc) if err != nil { - return skipReason, fmt.Errorf("cluster %s/%s assemble label selector failed, err: %v", ns, tcName, err) + return skipReason, err } - pvcs, err := rpc.pvcLister.PersistentVolumeClaims(ns).List(selector) + for _, pvc := range pvcs { + pvcName := pvc.GetName() + l := label.Label(pvc.Labels) + if !(l.IsPD() || l.IsTiKV()) { + skipReason[pvcName] = skipReasonPVCCleanerIsNotPDOrTiKV + continue + } + + if pvc.Status.Phase != corev1.ClaimBound { + // If pvc is not bound yet, it will not be processed + skipReason[pvcName] = skipReasonPVCCleanerPVCNotBound + continue + } + + if pvc.DeletionTimestamp != nil { + // PVC has been deleted, skip it + skipReason[pvcName] = skipReasonPVCCleanerPVCHasBeenDeleted + continue + } + + if len(pvc.Annotations[label.AnnPVCDeferDeleting]) == 0 { + // This pvc has not been marked as defer delete PVC, can't reclaim the PV bound to this PVC + skipReason[pvcName] = skipReasonPVCCleanerIsNotDeferDeletePVC + continue + } + + // PVC has been marked as defer delete PVC, try to reclaim the PV bound to this PVC + podName, exist := pvc.Annotations[label.AnnPodNameKey] + if !exist { + // PVC has not pod name annotation, this is an unexpected PVC, skip it + skipReason[pvcName] = skipReasonPVCCleanerPVCNotHasPodNameAnn + continue + } + + _, err := rpc.podLister.Pods(ns).Get(podName) + if err == nil { + // PVC is still referenced by this pod, can't reclaim PV + skipReason[pvcName] = skipReasonPVCCleanerPVCeferencedByPod + continue + } + if !errors.IsNotFound(err) { + return skipReason, fmt.Errorf("cluster %s/%s get pvc %s pod %s from local cache failed, err: %v", ns, tcName, pvcName, podName, err) + } + + // if pod not found in cache, re-check from apiserver directly to make sure the pod really not exist + _, err = rpc.kubeCli.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{}) + if err == nil { + // PVC is still referenced by this pod, can't reclaim PV + skipReason[pvcName] = skipReasonPVCCleanerPVCeferencedByPod + continue + } + if !errors.IsNotFound(err) { + return skipReason, fmt.Errorf("cluster %s/%s get pvc %s pod %s from apiserver failed, err: %v", ns, tcName, pvcName, podName, err) + } + + // Without pd or tikv pod reference this defer delete PVC, start to reclaim PV + pvName := pvc.Spec.VolumeName + pv, err := rpc.pvLister.Get(pvName) + if err != nil { + if errors.IsNotFound(err) { + skipReason[pvcName] = skipReasonPVCCleanerNotFoundPV + continue + } + return skipReason, fmt.Errorf("cluster %s/%s get pvc %s pv %s failed, err: %v", ns, tcName, pvcName, pvName, err) + } + + if pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimDelete { + err := rpc.pvControl.PatchPVReclaimPolicy(tc, pv, corev1.PersistentVolumeReclaimDelete) + if err != nil { + return skipReason, fmt.Errorf("cluster %s/%s patch pv %s to %s failed, err: %v", ns, tcName, pvName, corev1.PersistentVolumeReclaimDelete, err) + } + glog.Infof("cluster %s/%s patch pv %s to policy %s success", ns, tcName, pvName, corev1.PersistentVolumeReclaimDelete) + } + + apiPVC, err := rpc.kubeCli.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + skipReason[pvcName] = skipReasonPVCCleanerPVCNotFound + continue + } + return skipReason, fmt.Errorf("cluster %s/%s get pvc %s failed, err: %v", ns, tcName, pvcName, err) + } + + if apiPVC.UID != pvc.UID || apiPVC.ResourceVersion != pvc.ResourceVersion { + skipReason[pvcName] = skipReasonPVCCleanerPVCChanged + continue + } + + if err := rpc.pvcControl.DeletePVC(tc, pvc); err != nil { + return skipReason, fmt.Errorf("cluster %s/%s delete pvc %s failed, err: %v", ns, tcName, pvcName, err) + } + glog.Infof("cluster %s/%s reclaim pv %s success, pvc %s", ns, tcName, pvName, pvcName) + } + return skipReason, nil +} + +func (rpc *realPVCCleaner) cleanScheduleLock(tc *v1alpha1.TidbCluster) (map[string]string, error) { + ns := tc.GetNamespace() + tcName := tc.GetName() + // for unit test + skipReason := map[string]string{} + + pvcs, err := rpc.listAllPVCs(tc) if err != nil { - return skipReason, fmt.Errorf("cluster %s/%s list pvc failed, selector: %s, err: %v", ns, tcName, selector, err) + return skipReason, err } for _, pvc := range pvcs { @@ -98,8 +231,8 @@ func (rpc *realPVCCleaner) Clean(tc *v1alpha1.TidbCluster) (map[string]string, e podName, exist := pvc.Annotations[label.AnnPodNameKey] if !exist { - // waiting for pvc's meta info to be synced - skipReason[pvcName] = skipReasonPVCCleanerWaitingForPVCSync + // PVC has no pod name annotation, this is an unexpected PVC, skip it + skipReason[pvcName] = skipReasonPVCCleanerPVCNotHasPodNameAnn continue } @@ -136,6 +269,22 @@ func (rpc *realPVCCleaner) Clean(tc *v1alpha1.TidbCluster) (map[string]string, e return skipReason, nil } +func (rpc *realPVCCleaner) listAllPVCs(tc *v1alpha1.TidbCluster) ([]*corev1.PersistentVolumeClaim, error) { + ns := tc.GetNamespace() + tcName := tc.GetName() + + selector, err := label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).Selector() + if err != nil { + return nil, fmt.Errorf("cluster %s/%s assemble label selector failed, err: %v", ns, tcName, err) + } + + pvcs, err := rpc.pvcLister.PersistentVolumeClaims(ns).List(selector) + if err != nil { + return nil, fmt.Errorf("cluster %s/%s list pvc failed, selector: %s, err: %v", ns, tcName, selector, err) + } + return pvcs, nil +} + var _ PVCCleanerInterface = &realPVCCleaner{} type FakePVCCleaner struct { diff --git a/pkg/manager/member/pvc_cleaner_test.go b/pkg/manager/member/pvc_cleaner_test.go index e30f77efe6e..619527db355 100644 --- a/pkg/manager/member/pvc_cleaner_test.go +++ b/pkg/manager/member/pvc_cleaner_test.go @@ -17,18 +17,798 @@ import ( "fmt" "strings" "testing" + "time" . "github.com/onsi/gomega" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" kubeinformers "k8s.io/client-go/informers" kubefake "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" ) -func TestPVCCleanerClean(t *testing.T) { +func TestPVCCleanerReclaimPV(t *testing.T) { + g := NewGomegaWithT(t) + + tc := newTidbClusterForPD() + type testcase struct { + name string + pods []*corev1.Pod + apiPods []*corev1.Pod + pvcs []*corev1.PersistentVolumeClaim + apiPvcs []*corev1.PersistentVolumeClaim + pvs []*corev1.PersistentVolume + getPodFailed bool + patchPVFailed bool + getPVCFailed bool + deletePVCFailed bool + expectFn func(*GomegaWithT, map[string]string, *realPVCCleaner, error) + } + testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + + pcc, fakeCli, podIndexer, pvcIndexer, pvcControl, pvIndexer, pvControl := newFakePVCCleaner() + if test.pods != nil { + for _, pod := range test.pods { + podIndexer.Add(pod) + } + } + if test.apiPods != nil { + for _, apiPod := range test.apiPods { + fakeCli.CoreV1().Pods(apiPod.GetNamespace()).Create(apiPod) + } + } + if test.pvcs != nil { + for _, pvc := range test.pvcs { + pvcIndexer.Add(pvc) + } + } + if test.apiPvcs != nil { + for _, apiPvc := range test.apiPvcs { + fakeCli.CoreV1().PersistentVolumeClaims(apiPvc.GetNamespace()).Create(apiPvc) + } + } + if test.pvs != nil { + for _, pv := range test.pvs { + pvIndexer.Add(pv) + } + } + if test.getPodFailed { + fakeCli.PrependReactor("get", "pods", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("API server down") + }) + } + if test.patchPVFailed { + pvControl.SetUpdatePVError(fmt.Errorf("patch pv failed"), 0) + } + if test.getPVCFailed { + fakeCli.PrependReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("API server down") + }) + } + if test.deletePVCFailed { + pvcControl.SetDeletePVCError(fmt.Errorf("delete pvc failed"), 0) + } + + skipReason, err := pcc.reclaimPV(tc) + test.expectFn(g, skipReason, pcc, err) + } + tests := []testcase{ + { + name: "no pvcs", + pods: nil, + apiPods: nil, + pvcs: nil, + apiPvcs: nil, + pvs: nil, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(0)) + }, + }, + { + name: "not pd or tikv pvcs", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "tidb-test-tidb-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).TiDB().Labels(), + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: nil, + pvs: nil, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["tidb-test-tidb-0"]).To(Equal(skipReasonPVCCleanerIsNotPDOrTiKV)) + }, + }, + { + name: "pvc is not bound", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "tidb-test-tidb-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimPending, + }, + }, + }, + apiPvcs: nil, + pvs: nil, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["tidb-test-tidb-0"]).To(Equal(skipReasonPVCCleanerPVCNotBound)) + }, + }, + { + name: "pvc has been deleted", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "tidb-test-tidb-0", + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: nil, + pvs: nil, + getPodFailed: false, + patchPVFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["tidb-test-tidb-0"]).To(Equal(skipReasonPVCCleanerPVCHasBeenDeleted)) + }, + }, + { + name: "pvc is not defer delete pvc", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + pvs: nil, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["pd-test-pd-0"]).To(Equal(skipReasonPVCCleanerIsNotDeferDeletePVC)) + }, + }, + { + name: "pvc not has pod name annotation", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: nil, + pvs: nil, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["pd-test-pd-0"]).To(Equal(skipReasonPVCCleanerPVCNotHasPodNameAnn)) + }, + }, + { + name: "pvc is still referenced by pod,", + pods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pd-0", + Namespace: metav1.NamespaceDefault, + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + }, + }, + }, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: nil, + pvs: nil, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["pd-test-pd-0"]).To(Equal(skipReasonPVCCleanerPVCeferencedByPod)) + }, + }, + { + name: "not found pod that referenced pvc from local cache, but found this pod from apiserver", + pods: nil, + apiPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pd-0", + Namespace: metav1.NamespaceDefault, + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + }, + }, + }, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: nil, + pvs: nil, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["pd-test-pd-0"]).To(Equal(skipReasonPVCCleanerPVCeferencedByPod)) + }, + }, + { + name: "get pod from apiserver failed", + pods: nil, + apiPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pd-0", + Namespace: metav1.NamespaceDefault, + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + }, + }, + }, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: nil, + pvs: nil, + getPodFailed: true, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(0)) + g.Expect(strings.Contains(err.Error(), "API server down")).To(BeTrue()) + }, + }, + { + name: "not found pv bound to pvc", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: nil, + pvs: nil, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["pd-test-pd-0"]).To(Equal(skipReasonPVCCleanerNotFoundPV)) + }, + }, + { + name: "patch pv reclaim policy to delete policy failed", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: nil, + pvs: []*corev1.PersistentVolume{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolume", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-local-pv-0", + Namespace: metav1.NamespaceAll, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + }, + }, + }, + getPodFailed: false, + patchPVFailed: true, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(0)) + g.Expect(strings.Contains(err.Error(), "patch pv failed")).To(BeTrue()) + }, + }, + { + name: "found pvc from local cache, but not found this pvc from apiserver", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: nil, + pvs: []*corev1.PersistentVolume{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolume", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-local-pv-0", + Namespace: metav1.NamespaceAll, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + }, + }, + }, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["pd-test-pd-0"]).To(Equal(skipReasonPVCCleanerPVCNotFound)) + }, + }, + { + name: "get pvc from apiserver failed", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + UID: types.UID("pd-test"), + ResourceVersion: "1", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + UID: types.UID("pd-test"), + ResourceVersion: "1", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + pvs: []*corev1.PersistentVolume{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolume", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-local-pv-0", + Namespace: metav1.NamespaceAll, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + }, + }, + }, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: true, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(0)) + g.Expect(strings.Contains(err.Error(), "API server down")).To(BeTrue()) + }, + }, + { + name: "pvc has been changed", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + UID: types.UID("pd-test"), + ResourceVersion: "1", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + UID: types.UID("pd-test"), + ResourceVersion: "2", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + pvs: []*corev1.PersistentVolume{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolume", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-local-pv-0", + Namespace: metav1.NamespaceAll, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + }, + }, + }, + getPodFailed: false, + patchPVFailed: false, + getPVCFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(1)) + g.Expect(skipReason["pd-test-pd-0"]).To(Equal(skipReasonPVCCleanerPVCChanged)) + }, + }, + { + name: "delete pvc failed", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + UID: types.UID("pd-test"), + ResourceVersion: "1", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + UID: types.UID("pd-test"), + ResourceVersion: "1", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + pvs: []*corev1.PersistentVolume{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolume", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-local-pv-0", + Namespace: metav1.NamespaceAll, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + }, + }, + }, + getPodFailed: false, + patchPVFailed: false, + deletePVCFailed: true, + expectFn: func(g *GomegaWithT, skipReason map[string]string, pcc *realPVCCleaner, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(0)) + pv, pvGetErr := pcc.pvLister.Get("pd-local-pv-0") + g.Expect(pvGetErr).NotTo(HaveOccurred()) + g.Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(corev1.PersistentVolumeReclaimDelete)) + g.Expect(strings.Contains(err.Error(), "delete pvc failed")).To(BeTrue()) + }, + }, + { + name: "the defer delete pvc has been remove and reclaim pv success", + pods: nil, + apiPods: nil, + pvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + UID: types.UID("pd-test"), + ResourceVersion: "1", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + apiPvcs: []*corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "pd-test-pd-0", + UID: types.UID("pd-test"), + ResourceVersion: "1", + Labels: label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).PD().Labels(), + Annotations: map[string]string{ + label.AnnPVCDeferDeleting: time.Now().Format(time.RFC3339), + label.AnnPodNameKey: "test-pd-0", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pd-local-pv-0", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + }, + pvs: []*corev1.PersistentVolume{ + { + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolume", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-local-pv-0", + Namespace: metav1.NamespaceAll, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + }, + }, + }, + getPodFailed: false, + patchPVFailed: false, + deletePVCFailed: false, + expectFn: func(g *GomegaWithT, skipReason map[string]string, pcc *realPVCCleaner, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(skipReason)).To(Equal(0)) + pv, pvGetErr := pcc.pvLister.Get("pd-local-pv-0") + g.Expect(pvGetErr).NotTo(HaveOccurred()) + _, pvcGetErr := pcc.pvcLister.PersistentVolumeClaims(metav1.NamespaceAll).Get("pd-test-pd-0") + g.Expect(errors.IsNotFound(pvcGetErr)).To(BeTrue()) + g.Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(corev1.PersistentVolumeReclaimDelete)) + }, + }, + } + + for i := range tests { + testFn(&tests[i], t) + } +} + +func TestPVCCleanerCleanScheduleLock(t *testing.T) { g := NewGomegaWithT(t) tc := newTidbClusterForPD() @@ -42,7 +822,7 @@ func TestPVCCleanerClean(t *testing.T) { testFn := func(test *testcase, t *testing.T) { t.Log(test.name) - pcc, podIndexer, pvcIndexer, pvcControl := newFakePVCCleaner() + pcc, _, podIndexer, pvcIndexer, pvcControl, _, _ := newFakePVCCleaner() if test.pods != nil { for _, pod := range test.pods { podIndexer.Add(pod) @@ -57,7 +837,7 @@ func TestPVCCleanerClean(t *testing.T) { pvcControl.SetUpdatePVCError(fmt.Errorf("update PVC failed"), 0) } - skipReason, err := pcc.Clean(tc) + skipReason, err := pcc.cleanScheduleLock(tc) test.expectFn(g, skipReason, pcc, err) } tests := []testcase{ @@ -160,7 +940,7 @@ func TestPVCCleanerClean(t *testing.T) { }, }, { - name: "pvc's meta info has not to be synced", + name: "pvc not has pod name annotation", pods: nil, pvcs: []*corev1.PersistentVolumeClaim{ { @@ -178,7 +958,7 @@ func TestPVCCleanerClean(t *testing.T) { expectFn: func(g *GomegaWithT, skipReason map[string]string, _ *realPVCCleaner, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(skipReason)).To(Equal(1)) - g.Expect(skipReason["pd-test-pd-0"]).To(Equal(skipReasonPVCCleanerWaitingForPVCSync)) + g.Expect(skipReason["pd-test-pd-0"]).To(Equal(skipReasonPVCCleanerPVCNotHasPodNameAnn)) }, }, { @@ -347,13 +1127,22 @@ func TestPVCCleanerClean(t *testing.T) { } } -func newFakePVCCleaner() (*realPVCCleaner, cache.Indexer, cache.Indexer, *controller.FakePVCControl) { +func newFakePVCCleaner() (*realPVCCleaner, *kubefake.Clientset, cache.Indexer, cache.Indexer, *controller.FakePVCControl, cache.Indexer, *controller.FakePVControl) { kubeCli := kubefake.NewSimpleClientset() kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeCli, 0) podInformer := kubeInformerFactory.Core().V1().Pods() pvcInformer := kubeInformerFactory.Core().V1().PersistentVolumeClaims() pvcControl := controller.NewFakePVCControl(pvcInformer) + pvInformer := kubeInformerFactory.Core().V1().PersistentVolumes() + pvControl := controller.NewFakePVControl(pvInformer, pvcInformer) - return &realPVCCleaner{podInformer.Lister(), pvcControl, pvcInformer.Lister()}, - podInformer.Informer().GetIndexer(), pvcInformer.Informer().GetIndexer(), pvcControl + rpc := &realPVCCleaner{ + kubeCli, + podInformer.Lister(), + pvcControl, + pvcInformer.Lister(), + pvInformer.Lister(), + pvControl, + } + return rpc, kubeCli, podInformer.Informer().GetIndexer(), pvcInformer.Informer().GetIndexer(), pvcControl, pvInformer.Informer().GetIndexer(), pvControl } diff --git a/pkg/manager/meta/reclaim_policy_manager.go b/pkg/manager/meta/reclaim_policy_manager.go index 28e8b6f8a9a..54d20e4bf6b 100644 --- a/pkg/manager/meta/reclaim_policy_manager.go +++ b/pkg/manager/meta/reclaim_policy_manager.go @@ -55,6 +55,10 @@ func (rpm *reclaimPolicyManager) Sync(tc *v1alpha1.TidbCluster) error { if pvc.Spec.VolumeName == "" { continue } + if tc.Spec.EnablePVReclaim && len(pvc.Annotations[label.AnnPVCDeferDeleting]) != 0 { + // If the pv reclaim function is turned on, and when pv is the candidate pv to be reclaimed, skip patch this pv. + continue + } pv, err := rpm.pvLister.Get(pvc.Spec.VolumeName) if err != nil { return err diff --git a/pkg/manager/meta/reclaim_policy_manager_test.go b/pkg/manager/meta/reclaim_policy_manager_test.go index ad2ba13d36d..c9ebffc5080 100644 --- a/pkg/manager/meta/reclaim_policy_manager_test.go +++ b/pkg/manager/meta/reclaim_policy_manager_test.go @@ -14,9 +14,9 @@ package meta import ( - "testing" - "fmt" + "testing" + "time" . "github.com/onsi/gomega" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" @@ -34,12 +34,14 @@ import ( func TestReclaimPolicyManagerSync(t *testing.T) { g := NewGomegaWithT(t) type testcase struct { - name string - pvcHasLabels bool - pvcHasVolumeName bool - updateErr bool - err bool - changed bool + name string + pvcHasLabels bool + pvcHasVolumeName bool + updateErr bool + err bool + changed bool + enablePVRecalim bool + hasDeferDeleteAnn bool } testFn := func(test *testcase, t *testing.T) { @@ -54,6 +56,10 @@ func TestReclaimPolicyManagerSync(t *testing.T) { if !test.pvcHasVolumeName { pvc1.Spec.VolumeName = "" } + tc.Spec.EnablePVReclaim = test.enablePVRecalim + if test.hasDeferDeleteAnn { + pvc1.Annotations = map[string]string{label.AnnPVCDeferDeleting: time.Now().String()} + } rpm, fakePVControl, pvcIndexer, pvIndexer := newFakeReclaimPolicyManager() err := pvcIndexer.Add(pvc1) @@ -86,36 +92,54 @@ func TestReclaimPolicyManagerSync(t *testing.T) { tests := []testcase{ { - name: "normal", - pvcHasLabels: true, - pvcHasVolumeName: true, - updateErr: false, - err: false, - changed: true, + name: "normal", + pvcHasLabels: true, + pvcHasVolumeName: true, + updateErr: false, + err: false, + changed: true, + enablePVRecalim: false, + hasDeferDeleteAnn: false, + }, + { + name: "pvc don't have labels", + pvcHasLabels: false, + pvcHasVolumeName: true, + updateErr: false, + err: false, + changed: false, + enablePVRecalim: false, + hasDeferDeleteAnn: false, }, { - name: "pvc don't have labels", - pvcHasLabels: false, - pvcHasVolumeName: true, - updateErr: false, - err: false, - changed: false, + name: "pvc don't have volumeName", + pvcHasLabels: true, + pvcHasVolumeName: false, + updateErr: false, + err: false, + changed: false, + enablePVRecalim: false, + hasDeferDeleteAnn: false, }, { - name: "pvc don't have volumeName", - pvcHasLabels: false, - pvcHasVolumeName: false, - updateErr: false, - err: false, - changed: false, + name: "enable pv reclaim and pvc has defer delete annotation", + pvcHasLabels: true, + pvcHasVolumeName: true, + updateErr: false, + err: false, + changed: false, + enablePVRecalim: true, + hasDeferDeleteAnn: true, }, { - name: "update failed", - pvcHasLabels: true, - pvcHasVolumeName: true, - updateErr: true, - err: true, - changed: false, + name: "patch pv failed", + pvcHasLabels: true, + pvcHasVolumeName: true, + updateErr: true, + err: true, + changed: false, + enablePVRecalim: false, + hasDeferDeleteAnn: false, }, } diff --git a/tests/actions.go b/tests/actions.go index dba50784186..81338e51264 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -242,6 +242,7 @@ type TidbClusterConfig struct { BackupName string Namespace string ClusterName string + EnablePVReclaim bool OperatorTag string PDImage string TiKVImage string @@ -310,6 +311,7 @@ func (tc *TidbClusterConfig) TidbClusterHelmSetString(m map[string]string) strin set := map[string]string{ "clusterName": tc.ClusterName, + "enablePVReclaim": strconv.FormatBool(tc.EnablePVReclaim), "pd.storageClassName": tc.StorageClassName, "tikv.storageClassName": tc.StorageClassName, "tidb.storageClassName": tc.StorageClassName, @@ -936,6 +938,12 @@ func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterConfig) error return false, nil } } + if info.EnablePVReclaim { + glog.V(4).Infof("check reclaim pvs success when scale in pd or tikv") + if b, err := oa.checkReclaimPVSuccess(tc); !b && err == nil { + return false, nil + } + } return true, nil }); err != nil { glog.Errorf("check tidb cluster status failed: %s", err.Error()) @@ -1718,6 +1726,86 @@ func (oa *operatorActions) podsScheduleAnnHaveDeleted(tc *v1alpha1.TidbCluster) return true, nil } +func (oa *operatorActions) checkReclaimPVSuccess(tc *v1alpha1.TidbCluster) (bool, error) { + // check pv reclaim for pd + if err := oa.checkComponentReclaimPVSuccess(tc, label.PDLabelVal); err != nil { + glog.Errorf(err.Error()) + return false, nil + } + + // check pv reclaim for tikv + if err := oa.checkComponentReclaimPVSuccess(tc, label.TiKVLabelVal); err != nil { + glog.Errorf(err.Error()) + return false, nil + } + return true, nil +} + +func (oa *operatorActions) checkComponentReclaimPVSuccess(tc *v1alpha1.TidbCluster, component string) error { + ns := tc.GetNamespace() + tcName := tc.GetName() + + var replica int + switch component { + case label.PDLabelVal: + replica = int(tc.Spec.PD.Replicas) + case label.TiKVLabelVal: + replica = int(tc.Spec.TiKV.Replicas) + default: + return fmt.Errorf("check tidb cluster %s/%s component %s is not supported", ns, tcName, component) + } + + pvcList, err := oa.getComponentPVCList(tc, component) + if err != nil { + return err + } + + pvList, err := oa.getComponentPVList(tc, component) + if err != nil { + return err + } + + if len(pvcList) != replica || len(pvList) != replica { + return fmt.Errorf("tidb cluster %s/%s compoenent %s pv or pvc have not been reclaimed completely", ns, tcName, component) + } + + return nil +} + +func (oa *operatorActions) getComponentPVCList(tc *v1alpha1.TidbCluster, component string) ([]corev1.PersistentVolumeClaim, error) { + ns := tc.GetNamespace() + tcName := tc.GetName() + + listOptions := metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet( + label.New().Instance(tcName).Component(component).Labels()).String(), + } + + pvcList, err := oa.kubeCli.CoreV1().PersistentVolumeClaims(ns).List(listOptions) + if err != nil { + err := fmt.Errorf("failed to list pvcs for tidb cluster %s/%s, component: %s, err: %v", ns, tcName, component, err) + return nil, err + } + return pvcList.Items, nil +} + +func (oa *operatorActions) getComponentPVList(tc *v1alpha1.TidbCluster, component string) ([]corev1.PersistentVolume, error) { + ns := tc.GetNamespace() + tcName := tc.GetName() + + listOptions := metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet( + label.New().Instance(tcName).Component(component).Labels()).String(), + } + + pvList, err := oa.kubeCli.CoreV1().PersistentVolumes().List(listOptions) + if err != nil { + err := fmt.Errorf("failed to list pvs for tidb cluster %s/%s, component: %s, err: %v", ns, tcName, component, err) + return nil, err + } + return pvList.Items, nil +} + func (oa *operatorActions) storeLabelsIsSet(tc *v1alpha1.TidbCluster, topologyKey string) (bool, error) { pdCli := controller.GetPDClient(oa.pdControl, tc) for _, store := range tc.Status.TiKV.Stores { @@ -1963,7 +2051,9 @@ func getDatasourceID(svcName, namespace string) (int, error) { } defer func() { err := resp.Body.Close() - glog.Warning("close response failed", err) + if err != nil { + glog.Warningf("close response failed, err: %v", err) + } }() buf, err := ioutil.ReadAll(resp.Body) diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index b6d81958937..d4a20508a84 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -75,10 +75,14 @@ func main() { * This test case covers basic operators of a single cluster. * - deployment * - scaling + * - check reclaim pv success * - update configuration */ testBasic := func(wg *sync.WaitGroup, cluster *tests.TidbClusterConfig) { oa.CleanTidbClusterOrDie(cluster) + + // support reclaim pv when scale in tikv or pd component + cluster1.EnablePVReclaim = true oa.DeployTidbClusterOrDie(cluster) oa.CheckTidbClusterStatusOrDie(cluster) oa.CheckDisasterToleranceOrDie(cluster) @@ -255,6 +259,7 @@ func newTidbClusterConfig(ns, clusterName, password, tidbVersion string) *tests. return &tests.TidbClusterConfig{ Namespace: ns, ClusterName: clusterName, + EnablePVReclaim: false, OperatorTag: cfg.OperatorTag, PDImage: fmt.Sprintf("pingcap/pd:%s", tidbVersion), TiKVImage: fmt.Sprintf("pingcap/tikv:%s", tidbVersion),