Add switch to reclaim pv when scale in tikv or pd

.gitignore

@@ -10,6 +10,7 @@ tests/images/e2e/tidb-cluster/
 tests/images/e2e/tidb-backup/
 tests/images/e2e/tidb-operator/
 tests/images/e2e/manifests/
+manifests/crd-verify.yaml
 *.tar
 tmp/
 data/
@@ -30,4 +31,4 @@ coverage.out
 vendor
 tests/e2e/e2e.test
 .orig
-tkc
+tkctl

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -19,6 +19,7 @@ metadata:
     {{- end }}
 spec:
   pvReclaimPolicy: {{ .Values.pvReclaimPolicy }}
+  deferPVCDelete: {{ .Values.deferPVCDelete }}
   timezone: {{ .Values.timezone | default "UTC" }}
   enableTLSCluster: {{ .Values.enableTLSCluster | default false }}
   enableTLSClient: {{ .Values.enableTLSClient | default false }}

charts/tidb-cluster/values.yaml

@@ -22,11 +22,14 @@ schedulerName: tidb-scheduler
 # timezone is the default system timzone for TiDB
 timezone: UTC
 
-# reclaim policy of a PV, default: Retain.
+# reclaim policy of PV, default: Retain.
 # you must set it to Retain to ensure data safety in production environment.
 # https://pingcap.com/docs/v3.0/tidb-in-kubernetes/reference/configuration/local-pv/#data-security
 pvReclaimPolicy: Retain
 
+# defer delete policy of PVC, default: true
+deferPVCDelete: true
+
 # services is the service list to expose, default is ClusterIP
 # can be ClusterIP | NodePort | LoadBalancer
 services:

manifests/crd.yaml

@@ -74,6 +74,8 @@ spec:
           description: TidbClusterSpec describes the attributes that a user creates
             on a tidb cluster
           properties:
+            deferPVCDelete:
+              type: boolean
             enableTLSCluster:
               description: Enable TLS connection between TiDB server compoments
               type: boolean

pkg/apis/pingcap.com/v1alpha1/types.go

@@ -99,6 +99,7 @@ type TidbClusterSpec struct {
 	// Services list non-headless services type used in TidbCluster
 	Services        []Service                            `json:"services,omitempty"`
 	PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"`
+	DeferPVCDelete  bool                                 `json:"deferPVCDelete,omitempty"`
 	Timezone        string                               `json:"timezone,omitempty"`
 	// Enable TLS connection between TiDB server compoments
 	EnableTLSCluster bool `json:"enableTLSCluster,omitempty"`

pkg/controller/pv_control.go

@@ -200,7 +200,7 @@ func (fpc *FakePVControl) PatchPVReclaimPolicy(_ *v1alpha1.TidbCluster, pv *core
 	defer fpc.updatePVTracker.Inc()
 	if fpc.updatePVTracker.ErrorReady() {
 		defer fpc.updatePVTracker.Reset()
-		return fpc.updatePVTracker.err
+		return fpc.updatePVTracker.GetError()
 	}
 	pv.Spec.PersistentVolumeReclaimPolicy = reclaimPolicy
 
@@ -212,7 +212,7 @@ func (fpc *FakePVControl) UpdateMetaInfo(tc *v1alpha1.TidbCluster, pv *corev1.Pe
 	defer fpc.updatePVTracker.Inc()
 	if fpc.updatePVTracker.ErrorReady() {
 		defer fpc.updatePVTracker.Reset()
-		return nil, fpc.updatePVTracker.err
+		return nil, fpc.updatePVTracker.GetError()
 	}
 	ns := tc.GetNamespace()
 	pvcName := pv.Spec.ClaimRef.Name

pkg/controller/tidbcluster/tidb_cluster_controller.go

@@ -173,9 +173,12 @@ func NewController(
 				kubeCli,
 			),
 			mm.NewRealPVCCleaner(
+				kubeCli,
 				podInformer.Lister(),
 				pvcControl,
 				pvcInformer.Lister(),
+				pvInformer.Lister(),
+				pvControl,
 			),
 			recorder,
 		),

pkg/manager/member/pvc_cleaner.go

@@ -22,6 +22,8 @@ import (
 	"github.com/pingcap/tidb-operator/pkg/label"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	corelisters "k8s.io/client-go/listers/core/v1"
 )
 
@@ -31,7 +33,12 @@ const (
 	skipReasonPVCCleanerPVCNotHasLock            = "pvc cleaner: pvc not has schedule lock"
 	skipReasonPVCCleanerPodWaitingForScheduling  = "pvc cleaner: waiting for pod scheduling"
 	skipReasonPVCCleanerPodNotFound              = "pvc cleaner: the corresponding pod of pvc has not been found"
-	skipReasonPVCCleanerWaitingForPVCSync        = "pvc cleaner: waiting for pvc's meta info to be synced"
+	skipReasonPVCCleanerPVCNotBound              = "pvc cleaner: the pvc is not bound"
+	skipReasonPVCCleanerPVCNotHasPodNameAnn      = "pvc cleaner: pvc not has pod name annotation"
+	skipReasonPVCCleanerIsNotDeferDeletePVC      = "pvc cleaner: pvc has not been marked as defer delete pvc"
+	skipReasonPVCCleanerPVCeferencedByPod        = "pvc cleaner: pvc is still referenced by a pod"
+	skipReasonPVCCleanerNotFoundPV               = "pvc cleaner: not found pv bound to pvc"
+	skipReasonPVCCleanerPVWithDeletePolicy       = "pvc cleaner: pv bound to pvc has been set to delete policy"
 )
 
 // PVCCleaner implements the logic for cleaning the pvc related resource
@@ -40,37 +47,144 @@ type PVCCleanerInterface interface {
 }
 
 type realPVCCleaner struct {
+	kubeCli    kubernetes.Interface
 	podLister  corelisters.PodLister
 	pvcControl controller.PVCControlInterface
 	pvcLister  corelisters.PersistentVolumeClaimLister
+	pvLister   corelisters.PersistentVolumeLister
+	pvControl  controller.PVControlInterface
 }
 
 // NewRealPVCCleaner returns a realPVCCleaner
 func NewRealPVCCleaner(
+	kubeCli kubernetes.Interface,
 	podLister corelisters.PodLister,
 	pvcControl controller.PVCControlInterface,
-	pvcLister corelisters.PersistentVolumeClaimLister) PVCCleanerInterface {
+	pvcLister corelisters.PersistentVolumeClaimLister,
+	pvLister corelisters.PersistentVolumeLister,
+	pvControl controller.PVControlInterface) PVCCleanerInterface {
 	return &realPVCCleaner{
+		kubeCli,
 		podLister,
 		pvcControl,
 		pvcLister,
+		pvLister,
+		pvControl,
 	}
 }
 
 func (rpc *realPVCCleaner) Clean(tc *v1alpha1.TidbCluster) (map[string]string, error) {
+	if skipReason, err := rpc.cleanScheduleLock(tc); err != nil {
+		return skipReason, err
+	}
+
+	if tc.Spec.DeferPVCDelete {
+		// defer PVC delete policy is enabled, can't reclaim PV, so return directly.
+		return nil, nil
+	}
+
+	return rpc.reclaimPV(tc)
+}
+
+func (rpc *realPVCCleaner) reclaimPV(tc *v1alpha1.TidbCluster) (map[string]string, error) {
 	ns := tc.GetNamespace()
 	tcName := tc.GetName()
+
 	// for unit test
 	skipReason := map[string]string{}
 
-	selector, err := label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).Selector()
+	pvcs, err := rpc.listAllPVCs(tc)
 	if err != nil {
-		return skipReason, fmt.Errorf("cluster %s/%s assemble label selector failed, err: %v", ns, tcName, err)
+		return skipReason, err
 	}
 
-	pvcs, err := rpc.pvcLister.PersistentVolumeClaims(ns).List(selector)
+	for _, pvc := range pvcs {
+		pvcName := pvc.GetName()
+		l := label.Label(pvc.Labels)
+		if !(l.IsPD() || l.IsTiKV()) {
+			skipReason[pvcName] = skipReasonPVCCleanerIsNotPDOrTiKV
+			continue
+		}
+
+		if pvc.Status.Phase != corev1.ClaimBound {
+			// If pvc is not bound yet, it will not be processed
+			skipReason[pvcName] = skipReasonPVCCleanerPVCNotBound
+			continue
+		}
+
+		if len(pvc.Annotations[label.AnnPVCDeferDeleting]) == 0 {
+			// This pvc has not been marked as defer delete PVC, can't reclaim the PV bound to this PVC
+			skipReason[pvcName] = skipReasonPVCCleanerIsNotDeferDeletePVC
+			continue
+		}
+
+		// PVC has been marked as defer delete PVC, try to reclaim the PV bound to this PVC
+		podName, exist := pvc.Annotations[label.AnnPodNameKey]
+		if !exist {
+			// PVC has not pod name annotation, this is an unexpected PVC, skip it
+			skipReason[pvcName] = skipReasonPVCCleanerPVCNotHasPodNameAnn
+			continue
+		}
+
+		_, err := rpc.podLister.Pods(ns).Get(podName)
+		if err == nil {
+			// PVC is still referenced by this pod, can't reclaim PV
+			skipReason[pvcName] = skipReasonPVCCleanerPVCeferencedByPod
+			continue
+		}
+		if !errors.IsNotFound(err) {
+			return skipReason, fmt.Errorf("cluster %s/%s get pvc %s pod %s from local cache failed, err: %v", ns, tcName, pvcName, podName, err)
+		}
+
+		// if pod not found in cache, re-check from apiserver directly to make sure the pod really not exist
+		_, err = rpc.kubeCli.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
+		if err == nil {
+			// PVC is still referenced by this pod, can't reclaim PV
+			skipReason[pvcName] = skipReasonPVCCleanerPVCeferencedByPod
+			continue
+		}
+		if !errors.IsNotFound(err) {
+			return skipReason, fmt.Errorf("cluster %s/%s get pvc %s pod %s from apiserver failed, err: %v", ns, tcName, pvcName, podName, err)
+		}
+
+		// Without pd or tikv pod reference this defer delete PVC, start to reclaim PV
+		pvName := pvc.Spec.VolumeName
+		pv, err := rpc.pvLister.Get(pvName)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				skipReason[pvcName] = skipReasonPVCCleanerNotFoundPV
+				continue
+			}
+			return skipReason, fmt.Errorf("cluster %s/%s get pvc %s pv %s failed, err: %v", ns, tcName, pvcName, pvName, err)
+		}
+
+		if pv.Spec.PersistentVolumeReclaimPolicy == corev1.PersistentVolumeReclaimDelete {
+			skipReason[pvcName] = skipReasonPVCCleanerPVWithDeletePolicy
+			continue
+		}
+
+		err = rpc.pvControl.PatchPVReclaimPolicy(tc, pv, corev1.PersistentVolumeReclaimDelete)
+		if err != nil {
+			return skipReason, fmt.Errorf("cluster %s/%s patch pv %s to %s failed, err: %v", ns, tcName, pvName, corev1.PersistentVolumeReclaimDelete, err)
+		}
+
+		err = rpc.pvcControl.DeletePVC(tc, pvc)
+		if err != nil {
+			return skipReason, fmt.Errorf("cluster %s/%s delete pvc %s failed, err: %v", ns, tcName, pvcName, err)
+		}
+	}
+	return skipReason, nil
+}
+
+func (rpc *realPVCCleaner) cleanScheduleLock(tc *v1alpha1.TidbCluster) (map[string]string, error) {
+	ns := tc.GetNamespace()
+	tcName := tc.GetName()
+	// for unit test
+	skipReason := map[string]string{}
+
+	pvcs, err := rpc.listAllPVCs(tc)
 	if err != nil {
-		return skipReason, fmt.Errorf("cluster %s/%s list pvc failed, selector: %s, err: %v", ns, tcName, selector, err)
+		return skipReason, err
 	}
 
 	for _, pvc := range pvcs {
@@ -98,8 +212,8 @@ func (rpc *realPVCCleaner) Clean(tc *v1alpha1.TidbCluster) (map[string]string, e
 
 		podName, exist := pvc.Annotations[label.AnnPodNameKey]
 		if !exist {
-			// waiting for pvc's meta info to be synced
-			skipReason[pvcName] = skipReasonPVCCleanerWaitingForPVCSync
+			// PVC has not pod name annotation, this is an unexpected PVC, skip it
+			skipReason[pvcName] = skipReasonPVCCleanerPVCNotHasPodNameAnn
 			continue
 		}
 
@@ -136,6 +250,22 @@ func (rpc *realPVCCleaner) Clean(tc *v1alpha1.TidbCluster) (map[string]string, e
 	return skipReason, nil
 }
 
+func (rpc *realPVCCleaner) listAllPVCs(tc *v1alpha1.TidbCluster) ([]*corev1.PersistentVolumeClaim, error) {
+	ns := tc.GetNamespace()
+	tcName := tc.GetName()
+
+	selector, err := label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).Selector()
+	if err != nil {
+		return nil, fmt.Errorf("cluster %s/%s assemble label selector failed, err: %v", ns, tcName, err)
+	}
+
+	pvcs, err := rpc.pvcLister.PersistentVolumeClaims(ns).List(selector)
+	if err != nil {
+		return nil, fmt.Errorf("cluster %s/%s list pvc failed, selector: %s, err: %v", ns, tcName, selector, err)
+	}
+	return pvcs, nil
+}
+
 var _ PVCCleanerInterface = &realPVCCleaner{}
 
 type FakePVCCleaner struct {