diff --git a/changelogs/unreleased/7609-blackpiglet b/changelogs/unreleased/7609-blackpiglet new file mode 100644 index 00000000000..48644ae052a --- /dev/null +++ b/changelogs/unreleased/7609-blackpiglet @@ -0,0 +1 @@ +Merge CSI plugin code into Velero. \ No newline at end of file diff --git a/pkg/apis/velero/v1/labels_annotations.go b/pkg/apis/velero/v1/labels_annotations.go index 08b6b244053..99ef9dd0c20 100644 --- a/pkg/apis/velero/v1/labels_annotations.go +++ b/pkg/apis/velero/v1/labels_annotations.go @@ -97,6 +97,10 @@ const ( // VolumesToExcludeAnnotation is the annotation on a pod whose mounted volumes // should be excluded from pod volume backup. VolumesToExcludeAnnotation = "backup.velero.io/backup-volumes-excludes" + + // ExcludeFromBackupLabel is the label to exclude k8s resource from backup, + // even if the resource contains a matching selector label. + ExcludeFromBackupLabel = "velero.io/exclude-from-backup" ) type AsyncOperationIDPrefix string @@ -111,3 +115,44 @@ type VeleroResourceUsage string const ( VeleroResourceUsageDataUploadResult VeleroResourceUsage = "DataUpload" ) + +// CSI related plugin actions' constant variable +const ( + VolumeSnapshotLabel = "velero.io/volume-snapshot-name" + VolumeSnapshotHandleAnnotation = "velero.io/csi-volumesnapshot-handle" + VolumeSnapshotRestoreSize = "velero.io/vsi-volumesnapshot-restore-size" + CSIDriverNameAnnotation = "velero.io/csi-driver-name" + CSIDeleteSnapshotSecretName = "velero.io/csi-deletesnapshotsecret-name" + CSIDeleteSnapshotSecretNamespace = "velero.io/csi-deletesnapshotsecret-namespace" + CSIVSCDeletionPolicy = "velero.io/csi-vsc-deletion-policy" + VolumeSnapshotClassSelectorLabel = "velero.io/csi-volumesnapshot-class" + VolumeSnapshotClassDriverBackupAnnotationPrefix = "velero.io/csi-volumesnapshot-class" + VolumeSnapshotClassDriverPVCAnnotation = "velero.io/csi-volumesnapshot-class" + + // There is no release w/ these constants exported. Using the strings for now. + // CSI Labels volumesnapshotclass + // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L59-L60 + PrefixedSnapshotterListSecretNameKey = "csi.storage.k8s.io/snapshotter-list-secret-name" + PrefixedSnapshotterListSecretNamespaceKey = "csi.storage.k8s.io/snapshotter-list-secret-namespace" + + // CSI Labels volumesnapshotcontents + PrefixedSnapshotterSecretNameKey = "csi.storage.k8s.io/snapshotter-secret-name" + PrefixedSnapshotterSecretNamespaceKey = "csi.storage.k8s.io/snapshotter-secret-namespace" + + // Velero checks this annotation to determine whether to skip resource excluding check. + MustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" + // SkippedNoCSIPVAnnotation - Velero checks this annotation on processed PVC to + // find out if the snapshot was skipped b/c the PV is not provisioned via CSI + SkippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv" + + // DynamicPVRestoreLabel is the label key for dynamic PV restore + DynamicPVRestoreLabel = "velero.io/dynamic-pv-restore" + + // DataUploadNameAnnotation is the label key for the DataUpload name + DataUploadNameAnnotation = "velero.io/data-upload-name" +) + +/* + csiBIAPluginName = "velero.io/csi-pvc-backupper" + vsphereBIAPluginName = "velero.io/vsphere-pvc-backupper" +*/ diff --git a/pkg/backup/actions/csi_pvc_action.go b/pkg/backup/actions/csi_pvc_action.go new file mode 100644 index 00000000000..d2aaeed5897 --- /dev/null +++ b/pkg/backup/actions/csi_pvc_action.go @@ -0,0 +1,401 @@ +/* +Copyright 2019, 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package actions + +import ( + "context" + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/label" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" + uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// PVCBackupItemAction is a backup item action plugin for Velero. +type PVCBackupItemAction struct { + Log logrus.FieldLogger + Client kubernetes.Interface + SnapshotClient snapshotterClientSet.Interface + CRClient crclient.Client +} + +// AppliesTo returns information indicating that the PVCBackupItemAction should be invoked to backup PVCs. +func (p *PVCBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { + p.Log.Debug("PVCBackupItemAction AppliesTo") + + return velero.ResourceSelector{ + IncludedResources: []string{"persistentvolumeclaims"}, + }, nil +} + +// Execute recognizes PVCs backed by volumes provisioned by CSI drivers with volumesnapshotting capability and creates snapshots of the +// underlying PVs by creating volumesnapshot CSI API objects that will trigger the CSI driver to perform the snapshot operation on the volume. +func (p *PVCBackupItemAction) Execute(item runtime.Unstructured, backup *velerov1api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + p.Log.Info("Starting PVCBackupItemAction") + + // Do nothing if volume snapshots have not been requested in this backup + if boolptr.IsSetToFalse(backup.Spec.SnapshotVolumes) { + p.Log.Infof("Volume snapshotting not requested for backup %s/%s", backup.Namespace, backup.Name) + return item, nil, "", nil, nil + } + + if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + p.Log.WithFields( + logrus.Fields{ + "Backup": fmt.Sprintf("%s/%s", backup.Namespace, backup.Name), + "Phase": backup.Status.Phase, + }, + ).Debug("Backup is in finalizing phase. Skip this PVC.") + return item, nil, "", nil, nil + } + + var pvc corev1api.PersistentVolumeClaim + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), &pvc); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.Log.Debugf("Fetching underlying PV for PVC %s", fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name)) + // Do nothing if this is not a CSI provisioned volume + pv, err := csiutil.GetPVForPVC(&pvc, p.Client.CoreV1()) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + if pv.Spec.PersistentVolumeSource.CSI == nil { + p.Log.Infof("Skipping PVC %s/%s, associated PV %s is not a CSI volume", pvc.Namespace, pvc.Name, pv.Name) + + csiutil.AddAnnotations(&pvc.ObjectMeta, map[string]string{ + velerov1api.SkippedNoCSIPVAnnotation: "true", + }) + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + return &unstructured.Unstructured{Object: data}, nil, "", nil, err + } + + // Do nothing if FS uploader is used to backup this PV + isFSUploaderUsed, err := csiutil.IsPVCDefaultToFSBackup(pvc.Namespace, pvc.Name, p.Client.CoreV1(), boolptr.IsSetToTrue(backup.Spec.DefaultVolumesToFsBackup)) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + if isFSUploaderUsed { + p.Log.Infof("Skipping PVC %s/%s, PV %s will be backed up using FS uploader", pvc.Namespace, pvc.Name, pv.Name) + return item, nil, "", nil, nil + } + + // no storage class: we don't know how to map to a VolumeSnapshotClass + if pvc.Spec.StorageClassName == nil { + return item, nil, "", nil, errors.Errorf("Cannot snapshot PVC %s/%s, PVC has no storage class.", pvc.Namespace, pvc.Name) + } + + p.Log.Infof("Fetching storage class for PV %s", *pvc.Spec.StorageClassName) + storageClass, err := p.Client.StorageV1().StorageClasses().Get(context.TODO(), *pvc.Spec.StorageClassName, metav1.GetOptions{}) + if err != nil { + return nil, nil, "", nil, errors.Wrap(err, "error getting storage class") + } + p.Log.Debugf("Fetching volumesnapshot class for %s", storageClass.Provisioner) + snapshotClass, err := csiutil.GetVolumeSnapshotClass(storageClass.Provisioner, backup, &pvc, p.Log, p.SnapshotClient.SnapshotV1()) + if err != nil { + return nil, nil, "", nil, errors.Wrapf(err, "failed to get volumesnapshotclass for storageclass %s", storageClass.Name) + } + p.Log.Infof("volumesnapshot class=%s", snapshotClass.Name) + + vsLabels := map[string]string{} + for k, v := range pvc.ObjectMeta.Labels { + vsLabels[k] = v + } + vsLabels[velerov1api.BackupNameLabel] = label.GetValidName(backup.Name) + + // Craft the snapshot object to be created + snapshot := snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "velero-" + pvc.Name + "-", + Namespace: pvc.Namespace, + Labels: vsLabels, + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: &pvc.Name, + }, + VolumeSnapshotClassName: &snapshotClass.Name, + }, + } + + upd, err := p.SnapshotClient.SnapshotV1().VolumeSnapshots(pvc.Namespace).Create(context.TODO(), &snapshot, metav1.CreateOptions{}) + if err != nil { + return nil, nil, "", nil, errors.Wrapf(err, "error creating volume snapshot") + } + p.Log.Infof("Created volumesnapshot %s", fmt.Sprintf("%s/%s", upd.Namespace, upd.Name)) + + labels := map[string]string{ + velerov1api.VolumeSnapshotLabel: upd.Name, + velerov1api.BackupNameLabel: backup.Name, + } + + annotations := map[string]string{ + velerov1api.VolumeSnapshotLabel: upd.Name, + velerov1api.MustIncludeAdditionalItemAnnotation: "true", + } + + var additionalItems []velero.ResourceIdentifier + operationID := "" + var itemToUpdate []velero.ResourceIdentifier + + if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { + operationID = label.GetValidName(string(velerov1api.AsyncOperationIDPrefixDataUpload) + string(backup.UID) + "." + string(pvc.UID)) + dataUploadLog := p.Log.WithFields(logrus.Fields{ + "Source PVC": fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name), + "VolumeSnapshot": fmt.Sprintf("%s/%s", upd.Namespace, upd.Name), + "Operation ID": operationID, + "Backup": backup.Name, + }) + + // Wait until VS associated VSC snapshot handle created before returning with + // the Async operation for data mover. + _, err := csiutil.GetVolumeSnapshotContentForVolumeSnapshotImported(upd, p.SnapshotClient.SnapshotV1(), + dataUploadLog, true, backup.Spec.CSISnapshotTimeout.Duration) + if err != nil { + dataUploadLog.Errorf("Fail to wait VolumeSnapshot snapshot handle created: %s", err.Error()) + csiutil.CleanupVolumeSnapshot(upd, p.SnapshotClient.SnapshotV1(), p.Log) + return nil, nil, "", nil, errors.WithStack(err) + } + + dataUploadLog.Info("Starting data upload of backup") + + dataUpload, err := createDataUpload(context.Background(), backup, p.CRClient, upd, &pvc, operationID, snapshotClass) + if err != nil { + dataUploadLog.WithError(err).Error("failed to submit DataUpload") + csiutil.DeleteVolumeSnapshotIfAnyImported(context.Background(), p.SnapshotClient, *upd, dataUploadLog) + + return nil, nil, "", nil, errors.Wrapf(err, "error creating DataUpload") + } else { + itemToUpdate = []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "velero.io", + Resource: "datauploads", + }, + Namespace: dataUpload.Namespace, + Name: dataUpload.Name, + }, + } + // Set the DataUploadNameLabel, which is used for restore to let CSI plugin check whether + // it should handle the volume. If volume is CSI migration, PVC doesn't have the annotation. + annotations[velerov1api.DataUploadNameAnnotation] = dataUpload.Namespace + "/" + dataUpload.Name + + dataUploadLog.Info("DataUpload is submitted successfully.") + } + } else { + additionalItems = []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshots, + Namespace: upd.Namespace, + Name: upd.Name, + }, + } + } + + csiutil.AddAnnotations(&pvc.ObjectMeta, annotations) + csiutil.AddLabels(&pvc.ObjectMeta, labels) + + p.Log.Infof("Returning from PVCBackupItemAction with %d additionalItems to backup", len(additionalItems)) + for _, ai := range additionalItems { + p.Log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name) + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + return &unstructured.Unstructured{Object: pvcMap}, additionalItems, operationID, itemToUpdate, nil +} + +func (p *PVCBackupItemAction) Name() string { + return "PVCBackupItemAction" +} + +func (p *PVCBackupItemAction) Progress(operationID string, backup *velerov1api.Backup) (velero.OperationProgress, error) { + progress := velero.OperationProgress{} + if operationID == "" { + return progress, biav2.InvalidOperationIDError(operationID) + } + + dataUpload, err := getDataUpload(context.Background(), p.CRClient, operationID) + if err != nil { + p.Log.Errorf("fail to get DataUpload for backup %s/%s: %s", backup.Namespace, backup.Name, err.Error()) + return progress, err + } + if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseNew || dataUpload.Status.Phase == "" { + p.Log.Debugf("DataUpload is still not processed yet. Skip progress update.") + return progress, nil + } + + progress.Description = string(dataUpload.Status.Phase) + progress.OperationUnits = "Bytes" + progress.NCompleted = dataUpload.Status.Progress.BytesDone + progress.NTotal = dataUpload.Status.Progress.TotalBytes + + if dataUpload.Status.StartTimestamp != nil { + progress.Started = dataUpload.Status.StartTimestamp.Time + } + + if dataUpload.Status.CompletionTimestamp != nil { + progress.Updated = dataUpload.Status.CompletionTimestamp.Time + } + + if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCompleted { + progress.Completed = true + } else if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseFailed { + progress.Completed = true + progress.Err = dataUpload.Status.Message + } else if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCanceled { + progress.Completed = true + progress.Err = "DataUpload is canceled" + } + + return progress, nil +} + +func (p *PVCBackupItemAction) Cancel(operationID string, backup *velerov1api.Backup) error { + if operationID == "" { + return biav2.InvalidOperationIDError(operationID) + } + + dataUpload, err := getDataUpload(context.Background(), p.CRClient, operationID) + if err != nil { + p.Log.Errorf("fail to get DataUpload for backup %s/%s: %s", backup.Namespace, backup.Name, err.Error()) + return err + } + + return cancelDataUpload(context.Background(), p.CRClient, dataUpload) +} + +func newDataUpload(backup *velerov1api.Backup, vs *snapshotv1api.VolumeSnapshot, + pvc *corev1api.PersistentVolumeClaim, operationID string, vsClass *snapshotv1api.VolumeSnapshotClass) *velerov2alpha1.DataUpload { + dataUpload := &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + Kind: "DataUpload", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + GenerateName: backup.Name + "-", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: velerov1api.SchemeGroupVersion.String(), + Kind: "Backup", + Name: backup.Name, + UID: backup.UID, + Controller: boolptr.True(), + }, + }, + Labels: map[string]string{ + velerov1api.BackupNameLabel: label.GetValidName(backup.Name), + velerov1api.BackupUIDLabel: string(backup.UID), + velerov1api.PVCUIDLabel: string(pvc.UID), + velerov1api.AsyncOperationIDLabel: operationID, + }, + }, + Spec: velerov2alpha1.DataUploadSpec{ + SnapshotType: velerov2alpha1.SnapshotTypeCSI, + CSISnapshot: &velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: vs.Name, + StorageClass: *pvc.Spec.StorageClassName, + SnapshotClass: vsClass.Name, + }, + SourcePVC: pvc.Name, + DataMover: backup.Spec.DataMover, + BackupStorageLocation: backup.Spec.StorageLocation, + SourceNamespace: pvc.Namespace, + OperationTimeout: backup.Spec.CSISnapshotTimeout, + }, + } + + if backup.Spec.UploaderConfig != nil && backup.Spec.UploaderConfig.ParallelFilesUpload > 0 { + dataUpload.Spec.DataMoverConfig = make(map[string]string) + dataUpload.Spec.DataMoverConfig[uploaderUtil.ParallelFilesUpload] = fmt.Sprintf("%d", backup.Spec.UploaderConfig.ParallelFilesUpload) + } + + return dataUpload +} + +func createDataUpload(ctx context.Context, backup *velerov1api.Backup, crClient crclient.Client, + vs *snapshotv1api.VolumeSnapshot, pvc *corev1api.PersistentVolumeClaim, operationID string, + vsClass *snapshotv1api.VolumeSnapshotClass) (*velerov2alpha1.DataUpload, error) { + dataUpload := newDataUpload(backup, vs, pvc, operationID, vsClass) + + err := crClient.Create(ctx, dataUpload) + if err != nil { + return nil, errors.Wrap(err, "fail to create DataUpload CR") + } + + return dataUpload, err +} + +func getDataUpload(ctx context.Context, + crClient crclient.Client, operationID string) (*velerov2alpha1.DataUpload, error) { + dataUploadList := new(velerov2alpha1.DataUploadList) + err := crClient.List(ctx, dataUploadList, &crclient.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.AsyncOperationIDLabel: operationID}), + }) + if err != nil { + return nil, errors.Wrapf(err, "error to list DataUpload") + } + + if len(dataUploadList.Items) == 0 { + return nil, errors.Errorf("not found DataUpload for operationID %s", operationID) + } + + if len(dataUploadList.Items) > 1 { + return nil, errors.Errorf("more than one DataUpload found operationID %s", operationID) + } + + return &dataUploadList.Items[0], nil +} + +func cancelDataUpload(ctx context.Context, crClient crclient.Client, + dataUpload *velerov2alpha1.DataUpload) error { + + updatedDataUpload := dataUpload.DeepCopy() + updatedDataUpload.Spec.Cancel = true + + err := crClient.Patch(ctx, updatedDataUpload, crclient.MergeFrom(dataUpload)) + if err != nil { + return errors.Wrap(err, "error patch DataUpload") + } + + return nil +} diff --git a/pkg/backup/actions/csi_pvc_action_test.go b/pkg/backup/actions/csi_pvc_action_test.go new file mode 100644 index 00000000000..2ac3d56a701 --- /dev/null +++ b/pkg/backup/actions/csi_pvc_action_test.go @@ -0,0 +1,377 @@ +/* +Copyright 2019, 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package actions + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + v1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + snapshotfake "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/fake" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/fake" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" +) + +func TestExecute(t *testing.T) { + boolTrue := true + tests := []struct { + name string + backup *velerov1api.Backup + pvc *corev1.PersistentVolumeClaim + pv *corev1.PersistentVolume + sc *storagev1.StorageClass + vsClass *snapshotv1api.VolumeSnapshotClass + operationID string + expectedErr error + expectedBackup *velerov1api.Backup + expectedDataUpload *velerov2alpha1.DataUpload + expectedPVC *corev1.PersistentVolumeClaim + }{ + { + name: "Skip PVC handling if SnapshotVolume set to false", + backup: builder.ForBackup("velero", "test").SnapshotVolumes(false).Result(), + expectedErr: nil, + }, + { + name: "Skip PVC BIA when backup is in finalizing phase", + backup: builder.ForBackup("velero", "test").Phase(velerov1api.BackupPhaseFinalizing).Result(), + expectedErr: nil, + }, + { + name: "Test SnapshotMoveData", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + operationID: ".", + expectedErr: nil, + expectedDataUpload: &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + Namespace: "velero", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test", + velerov1api.BackupUIDLabel: "", + velerov1api.PVCUIDLabel: "", + velerov1api.AsyncOperationIDLabel: "du-.", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "velero.io/v1", + Kind: "Backup", + Name: "test", + UID: "", + Controller: &boolTrue, + }, + }, + }, + Spec: velerov2alpha1.DataUploadSpec{ + SnapshotType: velerov2alpha1.SnapshotTypeCSI, + CSISnapshot: &velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: "", + StorageClass: "testSC", + SnapshotClass: "testVSClass", + }, + SourcePVC: "testPVC", + SourceNamespace: "velero", + }, + }, + }, + { + name: "Verify PVC is modified as expected", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + operationID: ".", + expectedErr: nil, + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC"). + ObjectMeta(builder.WithAnnotations(velerov1api.MustIncludeAdditionalItemAnnotation, "true", velerov1api.DataUploadNameAnnotation, "velero/", velerov1api.VolumeSnapshotLabel, ""), + builder.WithLabels(velerov1api.BackupNameLabel, "test", velerov1api.VolumeSnapshotLabel, "")). + VolumeName("testPV").StorageClass("testSC").Phase(corev1.ClaimBound).Result(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + client := fake.NewSimpleClientset() + snapshotClient := snapshotfake.NewSimpleClientset() + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logrus.New() + logger.Level = logrus.DebugLevel + + if tc.pvc != nil { + _, err := client.CoreV1().PersistentVolumeClaims(tc.pvc.Namespace).Create(context.Background(), tc.pvc, metav1.CreateOptions{}) + require.NoError(t, err) + } + if tc.pv != nil { + _, err := client.CoreV1().PersistentVolumes().Create(context.Background(), tc.pv, metav1.CreateOptions{}) + require.NoError(t, err) + } + if tc.sc != nil { + _, err := client.StorageV1().StorageClasses().Create(context.Background(), tc.sc, metav1.CreateOptions{}) + require.NoError(t, err) + } + if tc.vsClass != nil { + _, err := snapshotClient.SnapshotV1().VolumeSnapshotClasses().Create(context.Background(), tc.vsClass, metav1.CreateOptions{}) + require.NoError(t, err) + } + + pvcBIA := PVCBackupItemAction{ + Log: logger, + Client: client, + SnapshotClient: snapshotClient, + CRClient: crClient, + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.pvc) + require.NoError(t, err) + + if boolptr.IsSetToTrue(tc.backup.Spec.SnapshotMoveData) == true { + go func() { + var vsList *v1.VolumeSnapshotList + err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) { + vsList, err = pvcBIA.SnapshotClient.SnapshotV1().VolumeSnapshots(tc.pvc.Namespace).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + if err != nil || len(vsList.Items) == 0 { + return false, nil + } + return true, nil + }) + + require.NoError(t, err) + vscName := "testVSC" + vsList.Items[0].Status = &v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName: &vscName} + _, err = pvcBIA.SnapshotClient.SnapshotV1().VolumeSnapshots(vsList.Items[0].Namespace).UpdateStatus(context.Background(), &vsList.Items[0], metav1.UpdateOptions{}) + require.NoError(t, err) + + handleName := "testHandle" + vsc := builder.ForVolumeSnapshotContent("testVSC").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &handleName}).Result() + _, err = pvcBIA.SnapshotClient.SnapshotV1().VolumeSnapshotContents().Create(context.Background(), vsc, metav1.CreateOptions{}) + require.NoError(t, err) + }() + } + + resultUnstructed, _, _, _, err := pvcBIA.Execute(&unstructured.Unstructured{Object: pvcMap}, tc.backup) + if tc.expectedErr != nil { + require.Equal(t, err, tc.expectedErr) + } + + if tc.expectedDataUpload != nil { + dataUploadList := new(velerov2alpha1.DataUploadList) + err := crClient.List(context.Background(), dataUploadList, &crclient.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: tc.backup.Name})}) + require.NoError(t, err) + require.Equal(t, 1, len(dataUploadList.Items)) + require.True(t, cmp.Equal(tc.expectedDataUpload, &dataUploadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion", "Name"))) + } + + if tc.expectedPVC != nil { + resultPVC := new(corev1.PersistentVolumeClaim) + runtime.DefaultUnstructuredConverter.FromUnstructured(resultUnstructed.UnstructuredContent(), resultPVC) + require.True(t, cmp.Equal(tc.expectedPVC, resultPVC, cmpopts.IgnoreFields(corev1.PersistentVolumeClaim{}, "Annotations"))) + } + }) + } +} + +func TestProgress(t *testing.T) { + currentTime := time.Now() + tests := []struct { + name string + backup *velerov1api.Backup + dataUpload *velerov2alpha1.DataUpload + operationID string + expectedErr string + expectedProgress velero.OperationProgress + }{ + { + name: "DataUpload cannot be found", + backup: builder.ForBackup("velero", "test").Result(), + operationID: "testing", + expectedErr: "not found DataUpload for operationID testing", + }, + { + name: "DataUpload is found", + backup: builder.ForBackup("velero", "test").Result(), + dataUpload: &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: "v2alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Status: velerov2alpha1.DataUploadStatus{ + Phase: velerov2alpha1.DataUploadPhaseFailed, + Progress: shared.DataMoveOperationProgress{ + BytesDone: 1000, + TotalBytes: 1000, + }, + StartTimestamp: &metav1.Time{Time: currentTime}, + CompletionTimestamp: &metav1.Time{Time: currentTime}, + Message: "Testing error", + }, + }, + operationID: "testing", + expectedProgress: velero.OperationProgress{ + Completed: true, + Err: "Testing error", + NCompleted: 1000, + NTotal: 1000, + OperationUnits: "Bytes", + Description: "Failed", + Started: currentTime, + Updated: currentTime, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + client := fake.NewSimpleClientset() + snapshotClient := snapshotfake.NewSimpleClientset() + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logrus.New() + + pvcBIA := PVCBackupItemAction{ + Log: logger, + Client: client, + SnapshotClient: snapshotClient, + CRClient: crClient, + } + + if tc.dataUpload != nil { + err := crClient.Create(context.Background(), tc.dataUpload) + require.NoError(t, err) + } + + progress, err := pvcBIA.Progress(tc.operationID, tc.backup) + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + } + require.True(t, cmp.Equal(tc.expectedProgress, progress, cmpopts.IgnoreFields(velero.OperationProgress{}, "Started", "Updated"))) + }) + } +} + +func TestCancel(t *testing.T) { + tests := []struct { + name string + backup *velerov1api.Backup + dataUpload velerov2alpha1.DataUpload + operationID string + expectedErr error + expectedDataUpload velerov2alpha1.DataUpload + }{ + { + name: "Cancel DataUpload", + backup: builder.ForBackup("velero", "test").Result(), + dataUpload: velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + }, + operationID: "testing", + expectedErr: nil, + expectedDataUpload: velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Spec: velerov2alpha1.DataUploadSpec{ + Cancel: true, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + client := fake.NewSimpleClientset() + snapshotClient := snapshotfake.NewSimpleClientset() + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logrus.New() + + pvcBIA := PVCBackupItemAction{ + Log: logger, + Client: client, + SnapshotClient: snapshotClient, + CRClient: crClient, + } + + err := crClient.Create(context.Background(), &tc.dataUpload) + require.NoError(t, err) + + err = pvcBIA.Cancel(tc.operationID, tc.backup) + if tc.expectedErr != nil { + require.Equal(t, err, tc.expectedErr) + } + + du := new(velerov2alpha1.DataUpload) + err = crClient.Get(context.Background(), crclient.ObjectKey{Namespace: tc.dataUpload.Namespace, Name: tc.dataUpload.Name}, du) + require.NoError(t, err) + + require.True(t, cmp.Equal(tc.expectedDataUpload, *du, cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion"))) + }) + } +} diff --git a/pkg/backup/actions/csi_volumesnapshot_action.go b/pkg/backup/actions/csi_volumesnapshot_action.go new file mode 100644 index 00000000000..38607dda94e --- /dev/null +++ b/pkg/backup/actions/csi_volumesnapshot_action.go @@ -0,0 +1,295 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package actions + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/label" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// VolumeSnapshotBackupItemAction is a backup item action plugin to backup +// CSI VolumeSnapshot objects using Velero +type VolumeSnapshotBackupItemAction struct { + Log logrus.FieldLogger +} + +// AppliesTo returns information indicating that the VolumeSnapshotBackupItemAction should be invoked to backup volumesnapshots. +func (p *VolumeSnapshotBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { + p.Log.Debug("VolumeSnapshotBackupItemAction AppliesTo") + + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute backs up a CSI volumesnapshot object and captures, as labels and annotations, information from its associated volumesnapshotcontents such as CSI driver name, storage snapshot handle +// and namespace and name of the snapshot delete secret, if any. It returns the volumesnapshotclass and the volumesnapshotcontents as additional items to be backed up. +func (p *VolumeSnapshotBackupItemAction) Execute(item runtime.Unstructured, backup *velerov1api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, + string, []velero.ResourceIdentifier, error) { + p.Log.Infof("Executing VolumeSnapshotBackupItemAction") + + var vs snapshotv1api.VolumeSnapshot + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), &vs); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + _, snapshotClient, err := csiutil.GetClients() + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + additionalItems := []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshotClasses, + Name: *vs.Spec.VolumeSnapshotClassName, + }, + } + + // determine if we are backing up a volumesnapshot that was created by velero while performing backup of a + // CSI backed PVC. + // For volumesnapshots that were created during the backup of a CSI backed PVC, we will wait for the volumecontents to + // be available. + // For volumesnapshots created outside of velero, we expect the volumesnapshotcontent to be available prior to backing up + // the volumesnapshot. In case of a failure, backup should be re-attempted after the CSI driver has reconciled the volumesnapshot. + // existence of the velerov1api.BackupNameLabel indicates that the volumesnapshot was created while backing up a + // CSI backed PVC. + + // We want to await reconciliation of only those volumesnapshots created during the ongoing backup. + // For this we will wait only if the backup label exists on the volumesnapshot object and the + // backup name is the same as that of the value of the backupLabel + backupOngoing := vs.Labels[velerov1api.BackupNameLabel] == label.GetValidName(backup.Name) + + p.Log.Infof("Getting VolumesnapshotContent for Volumesnapshot %s/%s", vs.Namespace, vs.Name) + + vsc, err := csiutil.GetVolumeSnapshotContentForVolumeSnapshotImported(&vs, snapshotClient.SnapshotV1(), p.Log, backupOngoing, backup.Spec.CSISnapshotTimeout.Duration) + if err != nil { + csiutil.CleanupVolumeSnapshot(&vs, snapshotClient.SnapshotV1(), p.Log) + return nil, nil, "", nil, errors.WithStack(err) + } + + if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + p.Log.WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). + WithField("BackupPhase", backup.Status.Phase).Debugf("Clean VolumeSnapshots.") + csiutil.DeleteVolumeSnapshot(vs, *vsc, backup, snapshotClient.SnapshotV1(), p.Log) + return item, nil, "", nil, nil + } + + annotations := make(map[string]string) + + if vsc != nil { + // when we are backing up volumesnapshots created outside of velero, we will not + // await volumesnapshot reconciliation and in this case GetVolumeSnapshotContentForVolumeSnapshot + // may not find the associated volumesnapshotcontents to add to the backup. + // This is not an error encountered in the backup process. So we add the volumesnapshotcontent + // to the backup only if one is found. + additionalItems = append(additionalItems, velero.ResourceIdentifier{ + GroupResource: kuberesource.VolumeSnapshotContents, + Name: vsc.Name, + }) + annotations[velerov1api.CSIVSCDeletionPolicy] = string(vsc.Spec.DeletionPolicy) + + if vsc.Status != nil { + if vsc.Status.SnapshotHandle != nil { + // Capture storage provider snapshot handle and CSI driver name + // to be used on restore to create a static volumesnapshotcontent that will be the source of the volumesnapshot. + annotations[velerov1api.VolumeSnapshotHandleAnnotation] = *vsc.Status.SnapshotHandle + annotations[velerov1api.CSIDriverNameAnnotation] = vsc.Spec.Driver + } + if vsc.Status.RestoreSize != nil { + annotations[velerov1api.VolumeSnapshotRestoreSize] = resource.NewQuantity(*vsc.Status.RestoreSize, resource.BinarySI).String() + } + } + + if backupOngoing { + p.Log.Infof("Patching volumesnapshotcontent %s with velero BackupNameLabel", vsc.Name) + // If we created the volumesnapshotcontent object during this ongoing backup, we would have created it with a DeletionPolicy of Retain. + // But, we want to retain these volumesnapshotcontent ONLY for the lifetime of the backup. To that effect, during velero backup + // deletion, we will update the DeletionPolicy of the volumesnapshotcontent and then delete the VolumeSnapshot object which will + // cascade delete the volumesnapshotcontent and the associated snapshot in the storage provider (handled by the CSI driver and + // the CSI common controller). + // However, in the event that the Volumesnapshot object is deleted outside of the backup deletion process, it is possible that + // the dynamically created volumesnapshotcontent object will be left as an orphaned and non-discoverable resource in the cluster as well + // as in the storage provider. To avoid piling up of such orphaned resources, we will want to discover and delete the dynamically created + // volumesnapshotcontents. We do that by adding the "velero.io/backup-name" label on the volumesnapshotcontent. + // Further, we want to add this label only on volumesnapshotcontents that were created during an ongoing velero backup. + + pb := []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, velerov1api.BackupNameLabel, label.GetValidName(backup.Name))) + if _, vscPatchError := snapshotClient.SnapshotV1().VolumeSnapshotContents().Patch(context.TODO(), vsc.Name, types.MergePatchType, pb, metav1.PatchOptions{}); vscPatchError != nil { + p.Log.Warnf("Failed to patch volumesnapshotcontent %s: %v", vsc.Name, vscPatchError) + } + } + } + + // Before applying the BIA v2, the in-cluster VS state is not persisted into backup. + // After the change, because the final state of VS will be stored in backup as the + // result of async operation result, need to patch the annotations into VS to work, + // because restore will check the annotations information. + pb := "{\"metadata\":{\"annotations\":{" + for k, v := range annotations { + pb += fmt.Sprintf("\"%s\":\"%s\",", k, v) + } + pb = strings.Trim(pb, ",") + pb += "}}}" + if _, err := snapshotClient.SnapshotV1().VolumeSnapshots(vs.Namespace).Patch(context.TODO(), + vs.Name, types.MergePatchType, []byte(pb), metav1.PatchOptions{}); err != nil { + p.Log.Errorf("Fail to patch volumesnapshot with content %s: %s.", pb, err.Error()) + return nil, nil, "", nil, errors.WithStack(err) + } + + annotations[velerov1api.MustIncludeAdditionalItemAnnotation] = "true" + // save newly applied annotations into the backed-up volumesnapshot item + csiutil.AddAnnotations(&vs.ObjectMeta, annotations) + + vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.Log.Infof("Returning from VolumeSnapshotBackupItemAction with %d additionalItems to backup", len(additionalItems)) + for _, ai := range additionalItems { + p.Log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name) + } + + operationID := "" + var itemToUpdate []velero.ResourceIdentifier + + // Only return Async operation for VSC created for this backup. + if backupOngoing { + // The operationID is of the form // + operationID = vs.Namespace + "/" + vs.Name + "/" + time.Now().Format(time.RFC3339) + itemToUpdate = []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshots, + Namespace: vs.Namespace, + Name: vs.Name, + }, + { + GroupResource: kuberesource.VolumeSnapshotContents, + Name: vsc.Name, + }, + } + } + + return &unstructured.Unstructured{Object: vsMap}, additionalItems, operationID, itemToUpdate, nil +} + +func (p *VolumeSnapshotBackupItemAction) Name() string { + return "VolumeSnapshotBackupItemAction" +} + +func (p *VolumeSnapshotBackupItemAction) Progress(operationID string, backup *velerov1api.Backup) (velero.OperationProgress, error) { + progress := velero.OperationProgress{} + if operationID == "" { + return progress, biav2.InvalidOperationIDError(operationID) + } + // The operationID is of the form // + operationIDParts := strings.Split(operationID, "/") + if len(operationIDParts) != 3 { + p.Log.Errorf("invalid operation ID %s", operationID) + return progress, biav2.InvalidOperationIDError(operationID) + } + var err error + if progress.Started, err = time.Parse(time.RFC3339, operationIDParts[2]); err != nil { + p.Log.Errorf("error parsing operation ID's StartedTime part into time %s: %s", operationID, err.Error()) + return progress, errors.WithStack(err) + } + + _, snapshotClient, err := csiutil.GetClients() + if err != nil { + return progress, errors.WithStack(err) + } + + vs, err := snapshotClient.SnapshotV1().VolumeSnapshots(operationIDParts[0]).Get( + context.Background(), operationIDParts[1], metav1.GetOptions{}) + if err != nil { + p.Log.Errorf("error getting volumesnapshot %s/%s: %s", operationIDParts[0], operationIDParts[1], err.Error()) + return progress, errors.WithStack(err) + } + + if vs.Status == nil { + p.Log.Debugf("VolumeSnapshot %s/%s has an empty status. Skip progress update.", vs.Namespace, vs.Name) + return progress, nil + } + + if boolptr.IsSetToTrue(vs.Status.ReadyToUse) { + p.Log.Debugf("VolumeSnapshot %s/%s is ReadyToUse. Continue on querying corresponding VolumeSnapshotContent.", + vs.Namespace, vs.Name) + } else if vs.Status.Error != nil { + errorMessage := "" + if vs.Status.Error.Message != nil { + errorMessage = *vs.Status.Error.Message + } + p.Log.Warnf("VolumeSnapshot has a temporary error %s. Snapshot controller will retry later.", errorMessage) + + return progress, nil + } + + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + vsc, err := snapshotClient.SnapshotV1().VolumeSnapshotContents().Get( + context.Background(), *vs.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + if err != nil { + p.Log.Errorf("error getting VolumeSnapshotContent %s: %s", *vs.Status.BoundVolumeSnapshotContentName, err.Error()) + return progress, errors.WithStack(err) + } + + if vsc.Status == nil { + p.Log.Debugf("VolumeSnapshotContent %s has an empty Status. Skip progress update.", vsc.Name) + return progress, nil + } + + now := time.Now() + + if boolptr.IsSetToTrue(vsc.Status.ReadyToUse) { + progress.Completed = true + progress.Updated = now + } else if vsc.Status.Error != nil { + progress.Completed = true + progress.Updated = now + if vsc.Status.Error.Message != nil { + progress.Err = *vsc.Status.Error.Message + } + p.Log.Warnf("VolumeSnapshotContent meets an error %s.", progress.Err) + } + } + + return progress, nil +} + +func (p *VolumeSnapshotBackupItemAction) Cancel(operationID string, backup *velerov1api.Backup) error { + // CSI Specification doesn't support canceling a snapshot creation. + return nil +} diff --git a/pkg/backup/actions/csi_volumesnapshotclass_action.go b/pkg/backup/actions/csi_volumesnapshotclass_action.go new file mode 100644 index 00000000000..9bb3775d1b2 --- /dev/null +++ b/pkg/backup/actions/csi_volumesnapshotclass_action.go @@ -0,0 +1,95 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package actions + +import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" + csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// VolumeSnapshotClassBackupItemAction is a backup item action plugin to backup +// CSI VolumeSnapshotclass objects using Velero +type VolumeSnapshotClassBackupItemAction struct { + Log logrus.FieldLogger +} + +// AppliesTo returns information indicating that the VolumeSnapshotClassBackupItemAction action should be invoked to backup volumesnapshotclass. +func (p *VolumeSnapshotClassBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { + p.Log.Debug("VolumeSnapshotClassBackupItemAction AppliesTo") + + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotclass.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute backs up a VolumeSnapshotClass object and returns as additional items any snapshot lister secret that may be referenced in its annotations. +func (p *VolumeSnapshotClassBackupItemAction) Execute(item runtime.Unstructured, backup *velerov1api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + p.Log.Infof("Executing VolumeSnapshotClassBackupItemAction") + + var snapClass snapshotv1api.VolumeSnapshotClass + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), &snapClass); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + additionalItems := []velero.ResourceIdentifier{} + if csiutil.IsVolumeSnapshotClassHasListerSecret(&snapClass) { + additionalItems = append(additionalItems, velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{Group: "", Resource: "secrets"}, + Name: snapClass.Annotations[velerov1api.PrefixedSnapshotterListSecretNameKey], + Namespace: snapClass.Annotations[velerov1api.PrefixedSnapshotterListSecretNamespaceKey], + }) + + csiutil.AddAnnotations(&snapClass.ObjectMeta, map[string]string{ + velerov1api.MustIncludeAdditionalItemAnnotation: "true", + }) + } + + snapClassMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&snapClass) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.Log.Infof("Returning from VolumeSnapshotClassBackupItemAction with %d additionalItems to backup", len(additionalItems)) + return &unstructured.Unstructured{Object: snapClassMap}, additionalItems, "", nil, nil +} + +func (p *VolumeSnapshotClassBackupItemAction) Name() string { + return "VolumeSnapshotClassBackupItemAction" +} + +func (p *VolumeSnapshotClassBackupItemAction) Progress(operationID string, backup *velerov1api.Backup) (velero.OperationProgress, error) { + progress := velero.OperationProgress{} + if operationID == "" { + return progress, biav2.InvalidOperationIDError(operationID) + } + + return progress, nil +} + +func (p *VolumeSnapshotClassBackupItemAction) Cancel(operationID string, backup *velerov1api.Backup) error { + return nil +} diff --git a/pkg/backup/actions/csi_volumesnapshotcontent_action.go b/pkg/backup/actions/csi_volumesnapshotcontent_action.go new file mode 100644 index 00000000000..aa67dc5f0da --- /dev/null +++ b/pkg/backup/actions/csi_volumesnapshotcontent_action.go @@ -0,0 +1,101 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package actions + +import ( + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// VolumeSnapshotContentBackupItemAction is a backup item action plugin to backup +// CSI VolumeSnapshotcontent objects using Velero +type VolumeSnapshotContentBackupItemAction struct { + Log logrus.FieldLogger +} + +// AppliesTo returns information indicating that the VolumeSnapshotContentBackupItemAction action should be invoked to backup volumesnapshotcontents. +func (p *VolumeSnapshotContentBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { + p.Log.Debug("VolumeSnapshotContentBackupItemAction AppliesTo") + + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotcontent.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute returns the unmodified volumesnapshotcontent object along with the snapshot deletion secret, if any, from its annotation +// as additional items to backup. +func (p *VolumeSnapshotContentBackupItemAction) Execute(item runtime.Unstructured, backup *velerov1api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + p.Log.Infof("Executing VolumeSnapshotContentBackupItemAction") + + if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + p.Log.WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). + WithField("BackupPhase", backup.Status.Phase).Debug("Skipping VolumeSnapshotContentBackupItemAction as backup is in finalizing phase.") + return item, nil, "", nil, nil + } + + var snapCont snapshotv1api.VolumeSnapshotContent + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), &snapCont); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + additionalItems := []velero.ResourceIdentifier{} + + // we should backup the snapshot deletion secrets that may be referenced in the volumesnapshotcontent's annotation + if csiutil.IsVolumeSnapshotContentHasDeleteSecret(&snapCont) { + // TODO: add GroupResource for secret into kuberesource + additionalItems = append(additionalItems, velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{Group: "", Resource: "secrets"}, + Name: snapCont.Annotations[velerov1api.PrefixedSnapshotterSecretNameKey], + Namespace: snapCont.Annotations[velerov1api.PrefixedSnapshotterSecretNamespaceKey], + }) + + csiutil.AddAnnotations(&snapCont.ObjectMeta, map[string]string{ + velerov1api.MustIncludeAdditionalItemAnnotation: "true", + }) + } + + snapContMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&snapCont) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.Log.Infof("Returning from VolumeSnapshotContentBackupItemAction with %d additionalItems to backup", len(additionalItems)) + return &unstructured.Unstructured{Object: snapContMap}, additionalItems, "", nil, nil +} + +func (p *VolumeSnapshotContentBackupItemAction) Name() string { + return "VolumeSnapshotContentBackupItemAction" +} + +func (p *VolumeSnapshotContentBackupItemAction) Progress(operationID string, backup *velerov1api.Backup) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +func (p *VolumeSnapshotContentBackupItemAction) Cancel(operationID string, backup *velerov1api.Backup) error { + // CSI Specification doesn't support canceling a snapshot creation. + return nil +} diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 0a8c7db7885..14257064862 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -383,16 +383,16 @@ func TestBackupOldResourceFiltering(t *testing.T) { Result(), apiResources: []*test.APIResource{ test.Pods( - builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), - builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), ), }, want: []string{ @@ -411,16 +411,16 @@ func TestBackupOldResourceFiltering(t *testing.T) { Result(), apiResources: []*test.APIResource{ test.Pods( - builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true", "a", "b")).Result(), + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), builder.ForPod("zoo", "raz").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), - builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true", "a", "b")).Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "b", "velero.io/exclude-from-backup", "true")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "b", velerov1.ExcludeFromBackupLabel, "true")).Result(), ), }, want: []string{ @@ -436,16 +436,16 @@ func TestBackupOldResourceFiltering(t *testing.T) { Result(), apiResources: []*test.APIResource{ test.Pods( - builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "false")).Result(), + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "false")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), - builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "1")).Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "1")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "")).Result(), ), }, want: []string{ @@ -1273,7 +1273,7 @@ func (a *recordResourcesAction) Execute(item runtime.Unstructured, backup *veler a.backups = append(a.backups, *backup) if a.skippedCSISnapshot { u := &unstructured.Unstructured{Object: item.UnstructuredContent()} - u.SetAnnotations(map[string]string{skippedNoCSIPVAnnotation: "true"}) + u.SetAnnotations(map[string]string{velerov1.SkippedNoCSIPVAnnotation: "true"}) item = u a.additionalItems = nil } @@ -2028,7 +2028,7 @@ func TestBackupActionAdditionalItems(t *testing.T) { builder.ForPod("ns-1", "pod-1").Result(), ), test.PVs( - builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 4b589fae318..953b3191e28 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -24,19 +24,17 @@ import ( "strings" "time" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" - kbClient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" @@ -58,11 +56,8 @@ import ( ) const ( - mustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" - skippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv" - excludeFromBackupLabel = "velero.io/exclude-from-backup" - csiBIAPluginName = "velero.io/csi-pvc-backupper" - vsphereBIAPluginName = "velero.io/vsphere-pvc-backupper" + csiBIAPluginName = "velero.io/csi-pvc-backupper" + vsphereBIAPluginName = "velero.io/vsphere-pvc-backupper" ) // itemBackupper can back up individual items to a tar writer. @@ -129,9 +124,9 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti if mustInclude { log.Infof("Skipping the exclusion checks for this resource") } else { - if metadata.GetLabels()[excludeFromBackupLabel] == "true" { - log.Infof("Excluding item because it has label %s=true", excludeFromBackupLabel) - ib.trackSkippedPV(obj, groupResource, "", fmt.Sprintf("item has label %s=true", excludeFromBackupLabel), log) + if metadata.GetLabels()[velerov1api.ExcludeFromBackupLabel] == "true" { + log.Infof("Excluding item because it has label %s=true", velerov1api.ExcludeFromBackupLabel) + ib.trackSkippedPV(obj, groupResource, "", fmt.Sprintf("item has label %s=true", velerov1api.ExcludeFromBackupLabel), log) return false, itemFiles, nil } // NOTE: we have to re-check namespace & resource includes/excludes because it's possible that @@ -384,18 +379,18 @@ func (ib *itemBackupper) executeActions( return nil, itemFiles, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) } u := &unstructured.Unstructured{Object: updatedItem.UnstructuredContent()} - if actionName == csiBIAPluginName && additionalItemIdentifiers == nil && u.GetAnnotations()[skippedNoCSIPVAnnotation] == "true" { + if actionName == csiBIAPluginName && additionalItemIdentifiers == nil && u.GetAnnotations()[velerov1api.SkippedNoCSIPVAnnotation] == "true" { // snapshot was skipped by CSI plugin ib.trackSkippedPV(obj, groupResource, csiSnapshotApproach, "skipped b/c it's not a CSI volume", log) - delete(u.GetAnnotations(), skippedNoCSIPVAnnotation) + delete(u.GetAnnotations(), velerov1api.SkippedNoCSIPVAnnotation) } else if (actionName == csiBIAPluginName || actionName == vsphereBIAPluginName) && !boolptr.IsSetToFalse(ib.backupRequest.Backup.Spec.SnapshotVolumes) { // the snapshot has been taken by the BIA plugin ib.unTrackSkippedPV(obj, groupResource, log) } - mustInclude := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation] == "true" || finalize + mustInclude := u.GetAnnotations()[velerov1api.MustIncludeAdditionalItemAnnotation] == "true" || finalize // remove the annotation as it's for communication between BIA and velero server, // we don't want the resource be restored with this annotation. - delete(u.GetAnnotations(), mustIncludeAdditionalItemAnnotation) + delete(u.GetAnnotations(), velerov1api.MustIncludeAdditionalItemAnnotation) obj = u // If async plugin started async operation, add it to the ItemOperations list diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index da915ed7e55..162d388f9e4 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -431,7 +431,7 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg // Add namespaces with label velero.io/exclude-from-backup=true into request.Spec.ExcludedNamespaces // Essentially, adding the label velero.io/exclude-from-backup=true to a namespace would be equivalent to setting spec.ExcludedNamespaces namespaces := corev1api.NamespaceList{} - if err := b.kbClient.List(context.Background(), &namespaces, kbclient.MatchingLabels{"velero.io/exclude-from-backup": "true"}); err == nil { + if err := b.kbClient.List(context.Background(), &namespaces, kbclient.MatchingLabels{velerov1api.ExcludeFromBackupLabel: "true"}); err == nil { for _, ns := range namespaces.Items { request.Spec.ExcludedNamespaces = append(request.Spec.ExcludedNamespaces, ns.Name) } diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 637d0805dd6..68dce2437bf 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -20,27 +20,22 @@ import ( "context" "time" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" - - "github.com/vmware-tanzu/velero/pkg/nodeagent" - "github.com/vmware-tanzu/velero/pkg/util/boolptr" - corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" - + "github.com/vmware-tanzu/velero/pkg/nodeagent" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/csi" "github.com/vmware-tanzu/velero/pkg/util/kube" - - snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - apierrors "k8s.io/apimachinery/pkg/api/errors" ) // CSISnapshotExposeParam define the input param for Expose of CSI snapshots diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index a9a88fd32b3..a55e2b1787b 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -20,25 +20,34 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" jsonpatch "github.com/evanphx/json-patch" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned" + snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - + "k8s.io/client-go/kubernetes" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/clientcmd" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/podvolume" "github.com/vmware-tanzu/velero/pkg/util/stringptr" "github.com/vmware-tanzu/velero/pkg/util/stringslice" - - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" - snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - apierrors "k8s.io/apimachinery/pkg/api/errors" ) const ( @@ -85,8 +94,12 @@ func WaitVolumeSnapshotReady(ctx context.Context, snapshotClient snapshotter.Sna return updated, err } -// GetVolumeSnapshotContentForVolumeSnapshot returns the volumesnapshotcontent object associated with the volumesnapshot -func GetVolumeSnapshotContentForVolumeSnapshot(volSnap *snapshotv1api.VolumeSnapshot, snapshotClient snapshotter.SnapshotV1Interface) (*snapshotv1api.VolumeSnapshotContent, error) { +// GetVolumeSnapshotContentForVolumeSnapshot returns the VolumeSnapshotContent +// object associated with the VolumeSnapshot. +func GetVolumeSnapshotContentForVolumeSnapshot( + volSnap *snapshotv1api.VolumeSnapshot, + snapshotClient snapshotter.SnapshotV1Interface, +) (*snapshotv1api.VolumeSnapshotContent, error) { if volSnap.Status == nil || volSnap.Status.BoundVolumeSnapshotContentName == nil { return nil, errors.Errorf("invalid snapshot info in volume snapshot %s", volSnap.Name) } @@ -112,8 +125,13 @@ func RetainVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interfa }) } -// DeleteVolumeSnapshotContentIfAny deletes a VSC by name if it exists, and log an error when the deletion fails -func DeleteVolumeSnapshotContentIfAny(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vscName string, log logrus.FieldLogger) { +// DeleteVolumeSnapshotContentIfAny deletes a VSC by name if it exists, +// and log an error when the deletion fails. +func DeleteVolumeSnapshotContentIfAny( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + vscName string, log logrus.FieldLogger, +) { err := snapshotClient.VolumeSnapshotContents().Delete(ctx, vscName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -124,7 +142,8 @@ func DeleteVolumeSnapshotContentIfAny(ctx context.Context, snapshotClient snapsh } } -// EnsureDeleteVS asserts the existence of a VS by name, deletes it and waits for its disappearance and returns errors on any failure +// EnsureDeleteVS asserts the existence of a VS by name, deletes it and waits for its +// disappearance and returns errors on any failure. func EnsureDeleteVS(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsName string, vsNamespace string, timeout time.Duration) error { err := snapshotClient.VolumeSnapshots(vsNamespace).Delete(ctx, vsName, metav1.DeleteOptions{}) @@ -176,7 +195,8 @@ func RemoveVSCProtect(ctx context.Context, snapshotClient snapshotter.SnapshotV1 return err } -// EnsureDeleteVSC asserts the existence of a VSC by name, deletes it and waits for its disappearance and returns errors on any failure +// EnsureDeleteVSC asserts the existence of a VSC by name, deletes it and waits for its +// disappearance and returns errors on any failure. func EnsureDeleteVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vscName string, timeout time.Duration) error { err := snapshotClient.VolumeSnapshotContents().Delete(ctx, vscName, metav1.DeleteOptions{}) @@ -215,8 +235,12 @@ func DeleteVolumeSnapshotIfAny(ctx context.Context, snapshotClient snapshotter.S } } -func patchVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, - vsc *snapshotv1api.VolumeSnapshotContent, updateFunc func(*snapshotv1api.VolumeSnapshotContent)) (*snapshotv1api.VolumeSnapshotContent, error) { +func patchVSC( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + vsc *snapshotv1api.VolumeSnapshotContent, + updateFunc func(*snapshotv1api.VolumeSnapshotContent), +) (*snapshotv1api.VolumeSnapshotContent, error) { origBytes, err := json.Marshal(vsc) if err != nil { return nil, errors.Wrap(err, "error marshaling original VSC") @@ -242,3 +266,489 @@ func patchVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interfac return patched, nil } + +func GetPVForPVC(pvc *corev1api.PersistentVolumeClaim, corev1 corev1client.PersistentVolumesGetter) (*corev1api.PersistentVolume, error) { + if pvc.Spec.VolumeName == "" { + return nil, errors.Errorf("PVC %s/%s has no volume backing this claim", pvc.Namespace, pvc.Name) + } + if pvc.Status.Phase != corev1api.ClaimBound { + // TODO: confirm if this PVC should be snapshotted if it has no PV bound + return nil, errors.Errorf("PVC %s/%s is in phase %v and is not bound to a volume", pvc.Namespace, pvc.Name, pvc.Status.Phase) + } + pvName := pvc.Spec.VolumeName + pv, err := corev1.PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "failed to get PV %s for PVC %s/%s", pvName, pvc.Namespace, pvc.Name) + } + return pv, nil +} + +func GetPodsUsingPVC(pvcNamespace, pvcName string, corev1 corev1client.PodsGetter) ([]corev1api.Pod, error) { + podsUsingPVC := []corev1api.Pod{} + podList, err := corev1.Pods(pvcNamespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + for _, p := range podList.Items { + for _, v := range p.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == pvcName { + podsUsingPVC = append(podsUsingPVC, p) + } + } + } + + return podsUsingPVC, nil +} + +func GetPodVolumeNameForPVC(pod corev1api.Pod, pvcName string) (string, error) { + for _, v := range pod.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == pvcName { + return v.Name, nil + } + } + return "", errors.Errorf("Pod %s/%s does not use PVC %s/%s", pod.Namespace, pod.Name, pod.Namespace, pvcName) +} + +func Contains(slice []string, key string) bool { + for _, i := range slice { + if i == key { + return true + } + } + return false +} + +func IsPVCDefaultToFSBackup(pvcNamespace, pvcName string, podClient corev1client.PodsGetter, defaultVolumesToFsBackup bool) (bool, error) { + pods, err := GetPodsUsingPVC(pvcNamespace, pvcName, podClient) + if err != nil { + return false, errors.WithStack(err) + } + + for _, p := range pods { + vols, _ := podvolume.GetVolumesByPod(&p, defaultVolumesToFsBackup, false) + if len(vols) > 0 { + volName, err := GetPodVolumeNameForPVC(p, pvcName) + if err != nil { + return false, err + } + if Contains(vols, volName) { + return true, nil + } + } + } + + return false, nil +} +func GetVolumeSnapshotClass(provisioner string, backup *velerov1api.Backup, pvc *corev1api.PersistentVolumeClaim, log logrus.FieldLogger, snapshotClient snapshotter.SnapshotV1Interface) (*snapshotv1api.VolumeSnapshotClass, error) { + snapshotClasses, err := snapshotClient.VolumeSnapshotClasses().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, errors.Wrap(err, "error listing volumesnapshot classes") + } + // If a snapshot class is sent for provider in PVC annotations, use that + snapshotClass, err := GetVolumeSnapshotClassFromPVCAnnotationsForDriver(pvc, provisioner, snapshotClasses) + if err != nil { + log.Debugf("Didn't find VolumeSnapshotClass from PVC annotations: %v", err) + } + if snapshotClass != nil { + return snapshotClass, nil + } + + // If there is no annotation in PVC, attempt to fetch it from backup annotations + snapshotClass, err = GetVolumeSnapshotClassFromBackupAnnotationsForDriver(backup, provisioner, snapshotClasses) + if err != nil { + log.Debugf("Didn't find VolumeSnapshotClass from Backup annotations: %v", err) + } + if snapshotClass != nil { + return snapshotClass, nil + } + + // fallback to default behaviour of fetching snapshot class based on label + snapshotClass, err = GetVolumeSnapshotClassForStorageClass(provisioner, snapshotClasses) + if err != nil || snapshotClass == nil { + return nil, errors.Wrap(err, "error getting volumesnapshotclass") + } + + return snapshotClass, nil +} + +func GetVolumeSnapshotClassFromPVCAnnotationsForDriver(pvc *corev1api.PersistentVolumeClaim, provisioner string, snapshotClasses *snapshotv1api.VolumeSnapshotClassList) (*snapshotv1api.VolumeSnapshotClass, error) { + annotationKey := velerov1api.VolumeSnapshotClassDriverPVCAnnotation + snapshotClassName, ok := pvc.ObjectMeta.Annotations[annotationKey] + if !ok { + return nil, nil + } + for _, sc := range snapshotClasses.Items { + if strings.EqualFold(snapshotClassName, sc.ObjectMeta.Name) { + if !strings.EqualFold(sc.Driver, provisioner) { + return nil, errors.Errorf("Incorrect volumesnapshotclass, snapshot class %s is not for driver %s", sc.ObjectMeta.Name, provisioner) + } + return &sc, nil + } + } + return nil, errors.Errorf("No CSI VolumeSnapshotClass found with name %s for provisioner %s for PVC %s", snapshotClassName, provisioner, pvc.Name) +} + +// GetVolumeSnapshotClassFromAnnotationsForDriver returns a VolumeSnapshotClass for the supplied volume provisioner/ driver name from the annotation of the backup +func GetVolumeSnapshotClassFromBackupAnnotationsForDriver(backup *velerov1api.Backup, provisioner string, snapshotClasses *snapshotv1api.VolumeSnapshotClassList) (*snapshotv1api.VolumeSnapshotClass, error) { + annotationKey := fmt.Sprintf("%s_%s", velerov1api.VolumeSnapshotClassDriverBackupAnnotationPrefix, strings.ToLower(provisioner)) + snapshotClassName, ok := backup.ObjectMeta.Annotations[annotationKey] + if !ok { + return nil, nil + } + for _, sc := range snapshotClasses.Items { + if strings.EqualFold(snapshotClassName, sc.ObjectMeta.Name) { + if !strings.EqualFold(sc.Driver, provisioner) { + return nil, errors.Errorf("Incorrect volumesnapshotclass, snapshot class %s is not for driver %s for backup %s", sc.ObjectMeta.Name, provisioner, backup.Name) + } + return &sc, nil + } + } + return nil, errors.Errorf("No CSI VolumeSnapshotClass found with name %s for driver %s for backup %s", snapshotClassName, provisioner, backup.Name) +} + +// GetVolumeSnapshotClassForStorageClass returns a VolumeSnapshotClass for the supplied volume provisioner/ driver name. +func GetVolumeSnapshotClassForStorageClass(provisioner string, snapshotClasses *snapshotv1api.VolumeSnapshotClassList) (*snapshotv1api.VolumeSnapshotClass, error) { + n := 0 + var vsclass snapshotv1api.VolumeSnapshotClass + // We pick the volumesnapshotclass that matches the CSI driver name and has a 'velero.io/csi-volumesnapshot-class' + // label. This allows multiple VolumesnapshotClasses for the same driver with different values for the + // other fields in the spec. + // https://github.com/kubernetes-csi/external-snapshotter/blob/release-4.2/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + for _, sc := range snapshotClasses.Items { + _, hasLabelSelector := sc.Labels[velerov1api.VolumeSnapshotClassSelectorLabel] + if sc.Driver == provisioner { + n += 1 + vsclass = sc + if hasLabelSelector { + return &sc, nil + } + } + } + // If there's only one volumesnapshotclass for the driver, return it. + if n == 1 { + return &vsclass, nil + } + return nil, errors.Errorf("failed to get volumesnapshotclass for provisioner %s, ensure that the desired volumesnapshot class has the %s label", provisioner, velerov1api.VolumeSnapshotClassSelectorLabel) +} + +// GetVolumeSnapshotContentForVolumeSnapshot returns the volumesnapshotcontent object associated with the volumesnapshot +func GetVolumeSnapshotContentForVolumeSnapshotImported(volSnap *snapshotv1api.VolumeSnapshot, snapshotClient snapshotter.SnapshotV1Interface, log logrus.FieldLogger, shouldWait bool, csiSnapshotTimeout time.Duration) (*snapshotv1api.VolumeSnapshotContent, error) { + if !shouldWait { + if volSnap.Status == nil || volSnap.Status.BoundVolumeSnapshotContentName == nil { + // volumesnapshot hasn't been reconciled and we're not waiting for it. + return nil, nil + } + vsc, err := snapshotClient.VolumeSnapshotContents().Get(context.TODO(), *volSnap.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "error getting volume snapshot content from API") + } + return vsc, nil + } + + // We'll wait 10m for the VSC to be reconciled polling every 5s unless csiSnapshotTimeout is set + timeout := 10 * time.Minute + if csiSnapshotTimeout > 0 { + timeout = csiSnapshotTimeout + } + interval := 5 * time.Second + var snapshotContent *snapshotv1api.VolumeSnapshotContent + + err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, true, func(ctx context.Context) (bool, error) { + vs, err := snapshotClient.VolumeSnapshots(volSnap.Namespace).Get(ctx, volSnap.Name, metav1.GetOptions{}) + if err != nil { + return false, errors.Wrapf(err, fmt.Sprintf("failed to get volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name)) + } + + if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil { + log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", volSnap.Namespace, volSnap.Name, interval/time.Second) + return false, nil + } + + snapshotContent, err = snapshotClient.VolumeSnapshotContents().Get(ctx, *vs.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + if err != nil { + return false, errors.Wrapf(err, fmt.Sprintf("failed to get volumesnapshotcontent %s for volumesnapshot %s/%s", *vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name)) + } + + // we need to wait for the VolumeSnaphotContent to have a snapshot handle because during restore, + // we'll use that snapshot handle as the source for the VolumeSnapshotContent so it's statically + // bound to the existing snapshot. + if snapshotContent.Status == nil || snapshotContent.Status.SnapshotHandle == nil { + log.Infof("Waiting for volumesnapshotcontents %s to have snapshot handle. Retrying in %ds", snapshotContent.Name, interval/time.Second) + if snapshotContent.Status != nil && snapshotContent.Status.Error != nil { + log.Warnf("Volumesnapshotcontent %s has error: %v", snapshotContent.Name, *snapshotContent.Status.Error.Message) + } + return false, nil + } + + return true, nil + }) + + if err != nil { + if err == wait.ErrorInterrupted(errors.New("timed out waiting for the condition")) { + if snapshotContent != nil && snapshotContent.Status != nil && snapshotContent.Status.Error != nil { + log.Errorf("Timed out awaiting reconciliation of volumesnapshot, Volumesnapshotcontent %s has error: %v", snapshotContent.Name, *snapshotContent.Status.Error.Message) + return nil, errors.Errorf("CSI got timed out with error: %v", *snapshotContent.Status.Error.Message) + } else { + log.Errorf("Timed out awaiting reconciliation of volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) + } + } + return nil, err + } + + return snapshotContent, nil +} + +func GetClients() (*kubernetes.Clientset, snapshotterClientSet.Interface, error) { + client, snapshotterClient, _, err := GetFullClients() + + return client, snapshotterClient, err +} + +func GetFullClients() (*kubernetes.Clientset, snapshotterClientSet.Interface, crclient.Client, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + clientConfig, err := kubeConfig.ClientConfig() + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + + client, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + + snapshotterClient, err := snapshotterClientSet.NewForConfig(clientConfig) + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + + scheme := runtime.NewScheme() + if err := velerov1api.AddToScheme(scheme); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if err := velerov2alpha1.AddToScheme(scheme); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + + crClient, err := crclient.New(clientConfig, crclient.Options{ + Scheme: scheme, + }) + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + + return client, snapshotterClient, crClient, nil +} + +// IsVolumeSnapshotClassHasListerSecret returns whether a volumesnapshotclass has a snapshotlister secret +func IsVolumeSnapshotClassHasListerSecret(vc *snapshotv1api.VolumeSnapshotClass) bool { + // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L59-L60 + // There is no release w/ these constants exported. Using the strings for now. + _, nameExists := vc.Annotations[velerov1api.PrefixedSnapshotterListSecretNameKey] + _, nsExists := vc.Annotations[velerov1api.PrefixedSnapshotterListSecretNamespaceKey] + return nameExists && nsExists +} + +// IsVolumeSnapshotContentHasDeleteSecret returns whether a volumesnapshotcontent has a deletesnapshot secret +func IsVolumeSnapshotContentHasDeleteSecret(vsc *snapshotv1api.VolumeSnapshotContent) bool { + // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L56-L57 + // use exported constants in the next release + _, nameExists := vsc.Annotations[velerov1api.PrefixedSnapshotterSecretNameKey] + _, nsExists := vsc.Annotations[velerov1api.PrefixedSnapshotterSecretNamespaceKey] + return nameExists && nsExists +} + +// IsVolumeSnapshotHasVSCDeleteSecret returns whether a volumesnapshot should set the deletesnapshot secret +// for the static volumesnapshotcontent that is created on restore +func IsVolumeSnapshotHasVSCDeleteSecret(vs *snapshotv1api.VolumeSnapshot) bool { + _, nameExists := vs.Annotations[velerov1api.CSIDeleteSnapshotSecretName] + _, nsExists := vs.Annotations[velerov1api.CSIDeleteSnapshotSecretNamespace] + return nameExists && nsExists +} + +// AddAnnotations adds the supplied key-values to the annotations on the object +func AddAnnotations(o *metav1.ObjectMeta, vals map[string]string) { + if o.Annotations == nil { + o.Annotations = make(map[string]string) + } + for k, v := range vals { + o.Annotations[k] = v + } +} + +// AddLabels adds the supplied key-values to the labels on the object +func AddLabels(o *metav1.ObjectMeta, vals map[string]string) { + if o.Labels == nil { + o.Labels = make(map[string]string) + } + for k, v := range vals { + o.Labels[k] = label.GetValidName(v) + } +} + +// IsVolumeSnapshotExists returns whether a specific volumesnapshot object exists. +func IsVolumeSnapshotExists(ns, name string, snapshotClient snapshotter.SnapshotV1Interface) bool { + vs, err := snapshotClient.VolumeSnapshots(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err == nil && vs != nil { + return true + } + + return false +} + +func SetVolumeSnapshotContentDeletionPolicy(vscName string, csiClient snapshotter.SnapshotV1Interface) error { + pb := []byte(`{"spec":{"deletionPolicy":"Delete"}}`) + _, err := csiClient.VolumeSnapshotContents().Patch(context.TODO(), vscName, types.MergePatchType, pb, metav1.PatchOptions{}) + + return err +} + +func HasBackupLabel(o *metav1.ObjectMeta, backupName string) bool { + if o.Labels == nil || len(strings.TrimSpace(backupName)) == 0 { + return false + } + return o.Labels[velerov1api.BackupNameLabel] == label.GetValidName(backupName) +} + +func CleanupVolumeSnapshot(volSnap *snapshotv1api.VolumeSnapshot, snapshotClient snapshotter.SnapshotV1Interface, log logrus.FieldLogger) { + log.Infof("Deleting Volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) + vs, err := snapshotClient.VolumeSnapshots(volSnap.Namespace).Get(context.TODO(), volSnap.Name, metav1.GetOptions{}) + if err != nil { + log.Debugf("Failed to get volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) + return + } + + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + // we patch the DeletionPolicy of the volumesnapshotcontent to set it to Delete. + // This ensures that the volume snapshot in the storage provider is also deleted. + err := SetVolumeSnapshotContentDeletionPolicy(*vs.Status.BoundVolumeSnapshotContentName, snapshotClient) + if err != nil { + log.Debugf("Failed to patch DeletionPolicy of volume snapshot %s/%s", vs.Namespace, vs.Name) + } + } + err = snapshotClient.VolumeSnapshots(vs.Namespace).Delete(context.TODO(), vs.Name, metav1.DeleteOptions{}) + if err != nil { + log.Debugf("Failed to delete volumesnapshot %s/%s: %v", vs.Namespace, vs.Name, err) + } else { + log.Infof("Deleted volumesnapshot with volumesnapshotContent %s/%s", vs.Namespace, vs.Name) + } +} + +// DeleteVolumeSnapshot is called by deleteVolumeSnapshots and handles the single VolumeSnapshot +// instance. +func DeleteVolumeSnapshot(vs snapshotv1api.VolumeSnapshot, vsc snapshotv1api.VolumeSnapshotContent, + backup *velerov1api.Backup, snapshotClient snapshotter.SnapshotV1Interface, logger logrus.FieldLogger) { + modifyVSCFlag := false + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil && len(*vs.Status.BoundVolumeSnapshotContentName) > 0 { + if vsc.Spec.DeletionPolicy == snapshotv1api.VolumeSnapshotContentDelete { + modifyVSCFlag = true + } + } else { + logger.Errorf("VolumeSnapshot %s/%s is not ready. This is not expected.", vs.Namespace, vs.Name) + } + + // Change VolumeSnapshotContent's DeletionPolicy to Retain before deleting VolumeSnapshot, + // because VolumeSnapshotContent will be deleted by deleting VolumeSnapshot, when + // DeletionPolicy is set to Delete, but Velero needs VSC for cleaning snapshot on cloud + // in backup deletion. + if modifyVSCFlag { + logger.Debugf("Patching VolumeSnapshotContent %s", vsc.Name) + patchData := []byte(fmt.Sprintf(`{"spec":{"deletionPolicy":"%s"}}`, snapshotv1api.VolumeSnapshotContentRetain)) + updatedVSC, err := snapshotClient.VolumeSnapshotContents().Patch(context.Background(), vsc.Name, types.MergePatchType, patchData, metav1.PatchOptions{}) + if err != nil { + logger.Errorf("fail to modify VolumeSnapshotContent %s DeletionPolicy to Retain: %s", vsc.Name, err.Error()) + return + } + + defer func() { + logger.Debugf("Start to recreate VolumeSnapshotContent %s", updatedVSC.Name) + err := recreateVolumeSnapshotContent(*updatedVSC, backup, snapshotClient, logger) + if err != nil { + logger.Errorf("fail to recreate VolumeSnapshotContent %s: %s", updatedVSC.Name, err.Error()) + } + }() + } + + // Delete VolumeSnapshot from cluster + logger.Debugf("Deleting VolumeSnapshot %s/%s", vs.Namespace, vs.Name) + err := snapshotClient.VolumeSnapshots(vs.Namespace).Delete(context.TODO(), vs.Name, metav1.DeleteOptions{}) + if err != nil { + logger.Errorf("fail to delete VolumeSnapshot %s/%s: %s", vs.Namespace, vs.Name, err.Error()) + } +} + +// recreateVolumeSnapshotContent will delete then re-create VolumeSnapshotContent, +// because some parameter in VolumeSnapshotContent Spec is immutable, e.g. VolumeSnapshotRef +// and Source. Source is updated to let csi-controller thinks the VSC is statically provsisioned with VS. +// Set VolumeSnapshotRef's UID to nil will let the csi-controller finds out the related VS is gone, then +// VSC can be deleted. +func recreateVolumeSnapshotContent(vsc snapshotv1api.VolumeSnapshotContent, backup *velerov1api.Backup, + snapshotClient snapshotter.SnapshotV1Interface, log logrus.FieldLogger) error { + // Read resource timeout from backup annotation, if not set, use default value. + timeout, err := time.ParseDuration(backup.Annotations[velerov1api.ResourceTimeoutAnnotation]) + if err != nil { + log.Warnf("fail to parse resource timeout annotation %s: %s", + backup.Annotations[velerov1api.ResourceTimeoutAnnotation], err.Error()) + timeout = 10 * time.Minute + } + log.Debugf("resource timeout is set to %s", timeout.String()) + interval := 1 * time.Second + + err = snapshotClient.VolumeSnapshotContents().Delete(context.TODO(), vsc.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.Wrapf(err, "fail to delete VolumeSnapshotContent: %s", vsc.Name) + } + + // Check VolumeSnapshotContents is already deleted, before re-creating it. + err = wait.PollUntilContextTimeout( + context.Background(), + interval, + timeout, + true, + func(ctx context.Context) (bool, error) { + _, err := snapshotClient.VolumeSnapshotContents().Get(ctx, vsc.Name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, errors.Wrapf(err, fmt.Sprintf("failed to get VolumeSnapshotContent %s", vsc.Name)) + } + return false, nil + }, + ) + if err != nil { + return errors.Wrapf(err, "fail to retrieve VolumeSnapshotContent %s info", vsc.Name) + } + + // Make the VolumeSnapshotContent static + vsc.Spec.Source = snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: vsc.Status.SnapshotHandle, + } + // Set VolumeSnapshotRef to none exist one, because VolumeSnapshotContent + // validation webhook will check whether name and namespace are nil. + // external-snapshotter needs Source pointing to snapshot and VolumeSnapshot + // reference's UID to nil to determine the VolumeSnapshotContent is deletable. + vsc.Spec.VolumeSnapshotRef = corev1api.ObjectReference{ + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + Kind: "VolumeSnapshot", + Namespace: "ns-" + string(vsc.UID), + Name: "name-" + string(vsc.UID), + } + // ResourceVersion shouldn't exist for new creation. + vsc.ResourceVersion = "" + _, err = snapshotClient.VolumeSnapshotContents().Create(context.TODO(), &vsc, metav1.CreateOptions{}) + if err != nil { + return errors.Wrapf(err, "fail to create VolumeSnapshotContent %s", vsc.Name) + } + + return nil +} + +func DeleteVolumeSnapshotIfAnyImported(ctx context.Context, snapshotClient snapshotterClientSet.Interface, + vs snapshotv1api.VolumeSnapshot, log logrus.FieldLogger) { + if err := snapshotClient.SnapshotV1().VolumeSnapshots(vs.Namespace).Delete(ctx, vs.Name, metav1.DeleteOptions{}); err != nil { + log.WithError(err).Warnf("fail to delete VolumeSnapshot %s/%s", vs.Namespace, vs.Name) + } +} diff --git a/pkg/util/csi/volume_snapshot_test.go b/pkg/util/csi/volume_snapshot_test.go index 0b02865e2d5..80230867318 100644 --- a/pkg/util/csi/volume_snapshot_test.go +++ b/pkg/util/csi/volume_snapshot_test.go @@ -24,13 +24,20 @@ import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/fake" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/logging" "github.com/vmware-tanzu/velero/pkg/util/stringptr" velerotest "github.com/vmware-tanzu/velero/pkg/test" @@ -201,7 +208,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) { } } -func TestGetVolumeSnapshotContentForVolumeSnapshot(t *testing.T) { +func TestGetVolumeSnapshotContentForVolumeSnapshotImported(t *testing.T) { vscName := "fake-vsc" vsObj := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ @@ -746,3 +753,1730 @@ func TestRemoveVSCProtect(t *testing.T) { }) } } + +var ( + csiStorageClass = "csi-hostpath-sc" +) + +func TestGetPVForPVC(t *testing.T) { + boundPVC := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-csi-pvc", + Namespace: "default", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + }, + } + matchingPV := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Spec: v1.PersistentVolumeSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + ClaimRef: &v1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: "test-csi-pvc", + Namespace: "default", + ResourceVersion: "1027", + UID: "7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: "hostpath.csi.k8s.io", + FSType: "ext4", + VolumeAttributes: map[string]string{ + "storage.kubernetes.io/csiProvisionerIdentity": "1582049697841-8081-hostpath.csi.k8s.io", + }, + VolumeHandle: "e61f2b48-527a-11ea-b54f-cab6317018f1", + }, + }, + PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimDelete, + StorageClassName: csiStorageClass, + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeBound, + }, + } + + pvcWithNoVolumeName := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "no-vol-pvc", + Namespace: "default", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + }, + Status: v1.PersistentVolumeClaimStatus{}, + } + + unboundPVC := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "unbound-pvc", + Namespace: "default", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimPending, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + }, + } + + testCases := []struct { + name string + inPVC *v1.PersistentVolumeClaim + expectError bool + expectedPV *v1.PersistentVolume + }{ + { + name: "should find PV matching the PVC", + inPVC: boundPVC, + expectError: false, + expectedPV: matchingPV, + }, + { + name: "should fail to find PV for PVC with no volumeName", + inPVC: pvcWithNoVolumeName, + expectError: true, + expectedPV: nil, + }, + { + name: "should fail to find PV for PVC not in bound phase", + inPVC: unboundPVC, + expectError: true, + expectedPV: nil, + }, + } + + objs := []runtime.Object{boundPVC, matchingPV, pvcWithNoVolumeName, unboundPVC} + fakeClient := fake.NewSimpleClientset(objs...) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualPV, actualError := GetPVForPVC(tc.inPVC, fakeClient.CoreV1()) + + if tc.expectError { + assert.NotNil(t, actualError, "Want error; Got nil error") + assert.Nilf(t, actualPV, "Want PV: nil; Got PV: %q", actualPV) + return + } + + assert.Nilf(t, actualError, "Want: nil error; Got: %v", actualError) + assert.Equalf(t, actualPV.Name, tc.expectedPV.Name, "Want PV with name %q; Got PV with name %q", tc.expectedPV.Name, actualPV.Name) + }) + } +} + +func TestGetPodsUsingPVC(t *testing.T) { + objs := []runtime.Object{ + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod3", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "awesome-ns", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + } + fakeClient := fake.NewSimpleClientset(objs...) + + testCases := []struct { + name string + pvcNamespace string + pvcName string + expectedPodCount int + }{ + { + name: "should find exactly 2 pods using the PVC", + pvcNamespace: "default", + pvcName: "csi-pvc1", + expectedPodCount: 2, + }, + { + name: "should find exactly 1 pod using the PVC", + pvcNamespace: "awesome-ns", + pvcName: "csi-pvc1", + expectedPodCount: 1, + }, + { + name: "should find 0 pods using the PVC", + pvcNamespace: "default", + pvcName: "unused-pvc", + expectedPodCount: 0, + }, + { + name: "should find 0 pods in non-existent namespace", + pvcNamespace: "does-not-exist", + pvcName: "csi-pvc1", + expectedPodCount: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualPods, err := GetPodsUsingPVC(tc.pvcNamespace, tc.pvcName, fakeClient.CoreV1()) + assert.Nilf(t, err, "Want error=nil; Got error=%v", err) + assert.Equalf(t, len(actualPods), tc.expectedPodCount, "unexpected number of pods in result; Want: %d; Got: %d", tc.expectedPodCount, len(actualPods)) + }) + } +} + +func TestGetPodVolumeNameForPVC(t *testing.T) { + testCases := []struct { + name string + pod v1.Pod + pvcName string + expectError bool + expectedVolumeName string + }{ + { + name: "should get volume name for pod with multuple PVCs", + pod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + { + Name: "csi-vol2", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc2", + }, + }, + }, + { + Name: "csi-vol3", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc3", + }, + }, + }, + }, + }, + }, + pvcName: "csi-pvc2", + expectedVolumeName: "csi-vol2", + expectError: false, + }, + { + name: "should get volume name from pod using exactly one PVC", + pod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + pvcName: "csi-pvc1", + expectedVolumeName: "csi-vol1", + expectError: false, + }, + { + name: "should return error for pod with no PVCs", + pod: v1.Pod{ + Spec: v1.PodSpec{}, + }, + pvcName: "csi-pvc2", + expectError: true, + }, + { + name: "should return error for pod with no matching PVC", + pod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + pvcName: "mismatch-pvc", + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVolumeName, err := GetPodVolumeNameForPVC(tc.pod, tc.pvcName) + if tc.expectError && err == nil { + assert.NotNil(t, err, "Want error; Got nil error") + return + } + assert.Equalf(t, tc.expectedVolumeName, actualVolumeName, "unexpected podVolumename returned. Want %s; Got %s", tc.expectedVolumeName, actualVolumeName) + }) + } +} + +func TestContains(t *testing.T) { + testCases := []struct { + name string + inSlice []string + inKey string + expectedResult bool + }{ + { + name: "should find the key", + inSlice: []string{"key1", "key2", "key3", "key4", "key5"}, + inKey: "key3", + expectedResult: true, + }, + { + name: "should not find the key in non-empty slice", + inSlice: []string{"key1", "key2", "key3", "key4", "key5"}, + inKey: "key300", + expectedResult: false, + }, + { + name: "should not find key in empty slice", + inSlice: []string{}, + inKey: "key300", + expectedResult: false, + }, + { + name: "should not find key in nil slice", + inSlice: nil, + inKey: "key300", + expectedResult: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualResult := Contains(tc.inSlice, tc.inKey) + assert.Equal(t, tc.expectedResult, actualResult) + }) + } +} + +func TestIsPVCDefaultToFSBackup(t *testing.T) { + objs := []runtime.Object{ + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default", + Annotations: map[string]string{ + "backup.velero.io/backup-volumes": "csi-vol1", + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod3", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-pod-1", + Namespace: "awesome-ns", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "awesome-csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-pod-2", + Namespace: "awesome-ns", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "awesome-csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "uploader-ns", + Annotations: map[string]string{ + "backup.velero.io/backup-volumes": "csi-vol1", + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "uploader-ns", + Annotations: map[string]string{ + "backup.velero.io/backup-volumes": "csi-vol1", + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + } + fakeClient := fake.NewSimpleClientset(objs...) + + testCases := []struct { + name string + inPVCNamespace string + inPVCName string + expectedIsFSUploaderUsed bool + defaultVolumesToFSBackup bool + }{ + { + name: "2 pods using PVC, 1 pod using uploader", + inPVCNamespace: "default", + inPVCName: "csi-pvc1", + expectedIsFSUploaderUsed: true, + defaultVolumesToFSBackup: false, + }, + { + name: "2 pods using PVC, 2 pods using uploader", + inPVCNamespace: "uploader-ns", + inPVCName: "csi-pvc1", + expectedIsFSUploaderUsed: true, + defaultVolumesToFSBackup: false, + }, + { + name: "2 pods using PVC, 0 pods using uploader", + inPVCNamespace: "awesome-ns", + inPVCName: "awesome-csi-pvc1", + expectedIsFSUploaderUsed: false, + defaultVolumesToFSBackup: false, + }, + { + name: "0 pods using PVC", + inPVCNamespace: "default", + inPVCName: "does-not-exist", + expectedIsFSUploaderUsed: false, + defaultVolumesToFSBackup: false, + }, + { + name: "2 pods using PVC, using uploader by default", + inPVCNamespace: "awesome-ns", + inPVCName: "awesome-csi-pvc1", + expectedIsFSUploaderUsed: true, + defaultVolumesToFSBackup: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualIsFSUploaderUsed, _ := IsPVCDefaultToFSBackup(tc.inPVCNamespace, tc.inPVCName, fakeClient.CoreV1(), tc.defaultVolumesToFSBackup) + assert.Equal(t, tc.expectedIsFSUploaderUsed, actualIsFSUploaderUsed) + }) + } +} + +func TestGetVolumeSnapshotClass(t *testing.T) { + // backups + backupFoo := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class_foo.csi.k8s.io": "foowithoutlabel", + }, + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + backupFoo2 := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo2", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class_foo.csi.k8s.io": "foo2", + }, + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + + backupBar2 := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class_bar.csi.k8s.io": "bar2", + }, + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + + backupNone := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "none", + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + + // pvcs + pvcFoo := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class": "foowithoutlabel", + }, + }, + Spec: v1.PersistentVolumeClaimSpec{}, + } + pvcFoo2 := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class": "foo2", + }, + }, + Spec: v1.PersistentVolumeClaimSpec{}, + } + pvcNone := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "none", + }, + Spec: v1.PersistentVolumeClaimSpec{}, + } + + // vsclasses + hostpathClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostpath", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "hostpath.csi.k8s.io", + } + + fooClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "foo.csi.k8s.io", + } + fooClassWithoutLabel := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foowithoutlabel", + }, + Driver: "foo.csi.k8s.io", + } + + barClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "true", + }, + }, + Driver: "bar.csi.k8s.io", + } + + barClass2 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar2", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "true", + }, + }, + Driver: "bar.csi.k8s.io", + } + + objs := []runtime.Object{hostpathClass, fooClass, barClass, fooClassWithoutLabel, barClass2} + fakeClient := snapshotFake.NewSimpleClientset(objs...) + + testCases := []struct { + name string + driverName string + pvc *v1.PersistentVolumeClaim + backup *velerov1api.Backup + expectedVSC *snapshotv1api.VolumeSnapshotClass + expectError bool + }{ + { + name: "no annotations on pvc and backup, should find hostpath volumesnapshotclass using default behaviour of labels", + driverName: "hostpath.csi.k8s.io", + pvc: pvcNone, + backup: backupNone, + expectedVSC: hostpathClass, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on pvc", + driverName: "foo.csi.k8s.io", + pvc: pvcFoo, + backup: backupNone, + expectedVSC: fooClassWithoutLabel, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on pvc, but csi driver does not match, no annotation on backup so fallback to default behaviour of labels", + driverName: "bar.csi.k8s.io", + pvc: pvcFoo, + backup: backupNone, + expectedVSC: barClass, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on pvc, but csi driver does not match so fallback to fetch from backupAnnotations ", + driverName: "bar.csi.k8s.io", + pvc: pvcFoo, + backup: backupBar2, + expectedVSC: barClass2, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on backup for foo.csi.k8s.io", + driverName: "foo.csi.k8s.io", + pvc: pvcNone, + backup: backupFoo, + expectedVSC: fooClassWithoutLabel, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on backup for bar.csi.k8s.io, no annotation corresponding to foo.csi.k8s.io, so fallback to default behaviour of labels", + driverName: "bar.csi.k8s.io", + pvc: pvcNone, + backup: backupFoo, + expectedVSC: barClass, + expectError: false, + }, + { + name: "no snapshotClass for given driver", + driverName: "blah.csi.k8s.io", + pvc: pvcNone, + backup: backupNone, + expectedVSC: nil, + expectError: true, + }, + { + name: "foo2 VSC annotations on pvc, but doesn't exist in cluster, fallback to default behaviour of labels", + driverName: "foo.csi.k8s.io", + pvc: pvcFoo2, + backup: backupFoo2, + expectedVSC: fooClass, + expectError: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualSnapshotClass, actualError := GetVolumeSnapshotClass(tc.driverName, tc.backup, tc.pvc, logrus.New(), fakeClient.SnapshotV1()) + if tc.expectError { + assert.NotNil(t, actualError) + assert.Nil(t, actualSnapshotClass) + return + } + assert.Equal(t, tc.expectedVSC, actualSnapshotClass) + }) + } +} +func TestGetVolumeSnapshotClassForStorageClass(t *testing.T) { + hostpathClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostpath", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "hostpath.csi.k8s.io", + } + + fooClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "foo.csi.k8s.io", + } + + barClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "bar.csi.k8s.io", + } + + bazClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + }, + Driver: "baz.csi.k8s.io", + } + + ambClass1 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amb1", + }, + Driver: "amb.csi.k8s.io", + } + + ambClass2 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amb2", + }, + Driver: "amb.csi.k8s.io", + } + + snapshotClasses := &snapshotv1api.VolumeSnapshotClassList{ + Items: []snapshotv1api.VolumeSnapshotClass{ + *hostpathClass, *fooClass, *barClass, *bazClass, *ambClass1, *ambClass2}, + } + + testCases := []struct { + name string + driverName string + expectedVSC *snapshotv1api.VolumeSnapshotClass + expectError bool + }{ + { + name: "should find hostpath volumesnapshotclass", + driverName: "hostpath.csi.k8s.io", + expectedVSC: hostpathClass, + expectError: false, + }, + { + name: "should find foo volumesnapshotclass", + driverName: "foo.csi.k8s.io", + expectedVSC: fooClass, + expectError: false, + }, + { + name: "should find bar volumesnapshotclass", + driverName: "bar.csi.k8s.io", + expectedVSC: barClass, + expectError: false, + }, + { + name: "should find baz volumesnapshotclass without \"velero.io/csi-volumesnapshot-class\" label, b/c there's only one vsclass matching the driver name", + driverName: "baz.csi.k8s.io", + expectedVSC: bazClass, + expectError: false, + }, + { + name: "should not find amb volumesnapshotclass without \"velero.io/csi-volumesnapshot-class\" label, b/c there're more than one vsclass matching the driver name", + driverName: "amb.csi.k8s.io", + expectedVSC: nil, + expectError: true, + }, + { + name: "should not find does-not-exist volumesnapshotclass", + driverName: "not-found.csi.k8s.io", + expectedVSC: nil, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVSC, actualError := GetVolumeSnapshotClassForStorageClass(tc.driverName, snapshotClasses) + + if tc.expectError { + assert.NotNil(t, actualError) + assert.Nil(t, actualVSC) + return + } + + assert.Equalf(t, tc.expectedVSC.Name, actualVSC.Name, "unexpected volumesnapshotclass name returned. Want: %s; Got:%s", tc.expectedVSC.Name, actualVSC.Name) + assert.Equalf(t, tc.expectedVSC.Driver, actualVSC.Driver, "unexpected driver name returned. Want: %s; Got:%s", tc.expectedVSC.Driver, actualVSC.Driver) + }) + } +} + +func TestGetVolumeSnapshotContentForVolumeSnapshot(t *testing.T) { + vscName := "snapcontent-7d1bdbd1-d10d-439c-8d8e-e1c2565ddc53" + snapshotHandle := "snapshot-handle" + vscObj := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: vscName, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: v1.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: &snapshotHandle, + }, + } + validVS := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + }, + } + + notFound := "does-not-exist" + vsWithVSCNotFound := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: notFound, + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: ¬Found, + }, + } + + vsWithNilStatus := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nil-status-vs", + Namespace: "default", + }, + Status: nil, + } + vsWithNilStatusField := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nil-status-field-vs", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: nil, + }, + } + + nilStatusVsc := "nil-status-vsc" + vscWithNilStatus := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: nilStatusVsc, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: v1.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: nil, + } + vsForNilStatusVsc := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-for-nil-status-vsc", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &nilStatusVsc, + }, + } + + nilStatusFieldVsc := "nil-status-field-vsc" + vscWithNilStatusField := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: nilStatusFieldVsc, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: v1.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: nil, + }, + } + vsForNilStatusFieldVsc := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-for-nil-status-field", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &nilStatusFieldVsc, + }, + } + + objs := []runtime.Object{vscObj, validVS, vsWithVSCNotFound, vsWithNilStatus, vsWithNilStatusField, vscWithNilStatus, vsForNilStatusVsc, vscWithNilStatusField, vsForNilStatusFieldVsc} + fakeClient := snapshotFake.NewSimpleClientset(objs...) + testCases := []struct { + name string + volSnap *snapshotv1api.VolumeSnapshot + exepctedVSC *snapshotv1api.VolumeSnapshotContent + wait bool + expectError bool + }{ + { + name: "waitEnabled should find volumesnapshotcontent for volumesnapshot", + volSnap: validVS, + exepctedVSC: vscObj, + wait: true, + expectError: false, + }, + { + name: "waitEnabled should not find volumesnapshotcontent for volumesnapshot with non-existing snapshotcontent name in status.BoundVolumeSnapshotContentName", + volSnap: vsWithVSCNotFound, + exepctedVSC: nil, + wait: true, + expectError: true, + }, + { + name: "waitEnabled should not find volumesnapshotcontent for a non-existent volumesnapshot", + wait: true, + exepctedVSC: nil, + expectError: true, + volSnap: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "not-found", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &nilStatusVsc, + }, + }, + }, + { + name: "waitDisabled should not find volumesnapshotcontent volumesnapshot status is nil", + wait: false, + expectError: false, + exepctedVSC: nil, + volSnap: vsWithNilStatus, + }, + { + name: "waitDisabled should not find volumesnapshotcontent volumesnapshot status.BoundVolumeSnapshotContentName is nil", + wait: false, + expectError: false, + exepctedVSC: nil, + volSnap: vsWithNilStatusField, + }, + { + name: "waitDisabled should find volumesnapshotcontent volumesnapshotcontent status is nil", + wait: false, + expectError: false, + exepctedVSC: vscWithNilStatus, + volSnap: vsForNilStatusVsc, + }, + { + name: "waitDisabled should find volumesnapshotcontent volumesnapshotcontent status.SnapshotHandle is nil", + wait: false, + expectError: false, + exepctedVSC: vscWithNilStatusField, + volSnap: vsForNilStatusFieldVsc, + }, + { + name: "waitDisabled should not find a non-existent volumesnapshotcontent", + wait: false, + exepctedVSC: nil, + expectError: true, + volSnap: vsWithVSCNotFound, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVSC, actualError := GetVolumeSnapshotContentForVolumeSnapshotImported(tc.volSnap, fakeClient.SnapshotV1(), logrus.New().WithField("fake", "test"), tc.wait, 0) + if tc.expectError && actualError == nil { + assert.NotNil(t, actualError) + assert.Nil(t, actualVSC) + return + } + assert.Equal(t, tc.exepctedVSC, actualVSC) + }) + } +} + +func TestIsVolumeSnapshotClassHasListerSecret(t *testing.T) { + testCases := []struct { + name string + snapClass snapshotv1api.VolumeSnapshotClass + expected bool + }{ + { + name: "should find both annotations", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-1", + Annotations: map[string]string{ + velerov1api.PrefixedSnapshotterListSecretNameKey: "snapListSecret", + velerov1api.PrefixedSnapshotterListSecretNamespaceKey: "awesome-ns", + }, + }, + }, + expected: true, + }, + { + name: "should not find both annotations name is missing", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-1", + Annotations: map[string]string{ + "foo": "snapListSecret", + velerov1api.PrefixedSnapshotterListSecretNamespaceKey: "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find both annotations namespace is missing", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-1", + Annotations: map[string]string{ + velerov1api.PrefixedSnapshotterListSecretNameKey: "snapListSecret", + "foo": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation non-empty annotation", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-2", + Annotations: map[string]string{ + "foo": "snapListSecret", + "bar": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation nil annotation", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-3", + Annotations: nil, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation empty annotation", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-3", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotClassHasListerSecret(&tc.snapClass) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestIsVolumeSnapshotContentHasDeleteSecret(t *testing.T) { + testCases := []struct { + name string + vsc snapshotv1api.VolumeSnapshotContent + expected bool + }{ + { + name: "should find both annotations", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Annotations: map[string]string{ + velerov1api.PrefixedSnapshotterSecretNameKey: "delSnapSecret", + velerov1api.PrefixedSnapshotterSecretNamespaceKey: "awesome-ns", + }, + }, + }, + expected: true, + }, + { + name: "should not find both annotations name is missing", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-2", + Annotations: map[string]string{ + "foo": "delSnapSecret", + velerov1api.PrefixedSnapshotterSecretNamespaceKey: "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find both annotations namespace is missing", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-3", + Annotations: map[string]string{ + velerov1api.PrefixedSnapshotterSecretNameKey: "delSnapSecret", + "foo": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation non-empty annotation", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-4", + Annotations: map[string]string{ + "foo": "delSnapSecret", + "bar": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation empty annotation", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-5", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation nil annotation", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-6", + Annotations: nil, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotContentHasDeleteSecret(&tc.vsc) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestIsVolumeSnapshotHasVSCDeleteSecret(t *testing.T) { + testCases := []struct { + name string + vs snapshotv1api.VolumeSnapshot + expected bool + }{ + { + name: "should find both annotations", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "velero.io/csi-deletesnapshotsecret-name": "snapDelSecret", + "velero.io/csi-deletesnapshotsecret-namespace": "awesome-ns", + }, + }, + }, + expected: true, + }, + { + name: "should not find both annotations name is missing", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "foo": "snapDelSecret", + "velero.io/csi-deletesnapshotsecret-namespace": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find both annotations namespace is missing", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "velero.io/csi-deletesnapshotsecret-name": "snapDelSecret", + "foo": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find annotation non-empty annotation", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "foo": "snapDelSecret", + "bar": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find annotation empty annotation", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + { + name: "should not find annotation nil annotation", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: nil, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotHasVSCDeleteSecret(&tc.vs) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestAddAnnotations(t *testing.T) { + annotationValues := map[string]string{ + "k1": "v1", + "k2": "v2", + "k3": "v3", + "k4": "v4", + "k5": "v5", + } + testCases := []struct { + name string + o metav1.ObjectMeta + toAdd map[string]string + }{ + { + name: "should create a new annotation map when annotation is nil", + o: metav1.ObjectMeta{ + Annotations: nil, + }, + toAdd: annotationValues, + }, + { + name: "should add all supplied annotations into empty annotation", + o: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + toAdd: annotationValues, + }, + { + name: "should add all supplied annotations to existing annotation", + o: metav1.ObjectMeta{ + Annotations: map[string]string{ + "k100": "v100", + "k200": "v200", + "k300": "v300", + }, + }, + toAdd: annotationValues, + }, + { + name: "should overwrite some existing annotations", + o: metav1.ObjectMeta{ + Annotations: map[string]string{ + "k100": "v100", + "k2": "v200", + "k300": "v300", + }, + }, + toAdd: annotationValues, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + AddAnnotations(&tc.o, tc.toAdd) + for k, v := range tc.toAdd { + actual, exists := tc.o.Annotations[k] + assert.True(t, exists) + assert.Equal(t, v, actual) + } + }) + } +} + +func TestAddLabels(t *testing.T) { + labelValues := map[string]string{ + "l1": "v1", + "l2": "v2", + "l3": "v3", + "l4": "v4", + "l5": "v5", + } + testCases := []struct { + name string + o metav1.ObjectMeta + toAdd map[string]string + }{ + { + name: "should create a new labels map when labels is nil", + o: metav1.ObjectMeta{ + Labels: nil, + }, + toAdd: labelValues, + }, + { + name: "should add all supplied labels into empty labels", + o: metav1.ObjectMeta{ + Labels: map[string]string{}, + }, + toAdd: labelValues, + }, + { + name: "should add all supplied labels to existing labels", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "l100": "v100", + "l200": "v200", + "l300": "v300", + }, + }, + toAdd: labelValues, + }, + { + name: "should overwrite some existing labels", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + toAdd: labelValues, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + AddLabels(&tc.o, tc.toAdd) + for k, v := range tc.toAdd { + actual, exists := tc.o.Labels[k] + assert.True(t, exists) + assert.Equal(t, v, actual) + } + }) + } +} + +func TestIsVolumeSnapshotExists(t *testing.T) { + vsExists := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-exists", + Namespace: "default", + }, + } + vsNotExists := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-does-not-exists", + Namespace: "default", + }, + } + + objs := []runtime.Object{vsExists} + fakeClient := snapshotFake.NewSimpleClientset(objs...) + testCases := []struct { + name string + expected bool + vs *snapshotv1api.VolumeSnapshot + }{ + { + name: "should find existing VolumeSnapshot object", + expected: true, + vs: vsExists, + }, + { + name: "should not find non-existing VolumeSnapshot object", + expected: false, + vs: vsNotExists, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotExists(tc.vs.Namespace, tc.vs.Name, fakeClient.SnapshotV1()) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestSetVolumeSnapshotContentDeletionPolicy(t *testing.T) { + testCases := []struct { + name string + inputVSCName string + objs []runtime.Object + expectError bool + }{ + { + name: "should update DeletionPolicy of a VSC from retain to delete", + inputVSCName: "retainVSC", + objs: []runtime.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "retainVSC", + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + }, + }, + }, + expectError: false, + }, + { + name: "should be a no-op updating if DeletionPolicy of a VSC is already Delete", + inputVSCName: "deleteVSC", + objs: []runtime.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deleteVSC", + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, + }, + }, + }, + expectError: false, + }, + { + name: "should update DeletionPolicy of a VSC with no DeletionPolicy", + inputVSCName: "nothingVSC", + objs: []runtime.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nothingVSC", + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{}, + }, + }, + expectError: false, + }, + { + name: "should return not found error if supplied VSC does not exist", + inputVSCName: "does-not-exist", + objs: []runtime.Object{}, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := snapshotFake.NewSimpleClientset(tc.objs...) + err := SetVolumeSnapshotContentDeletionPolicy(tc.inputVSCName, fakeClient.SnapshotV1()) + if tc.expectError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + actual, err := fakeClient.SnapshotV1().VolumeSnapshotContents().Get(context.TODO(), tc.inputVSCName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, snapshotv1api.VolumeSnapshotContentDelete, actual.Spec.DeletionPolicy) + } + }) + } +} + +func TestIsByBackup(t *testing.T) { + testCases := []struct { + name string + o metav1.ObjectMeta + backupName string + expected bool + }{ + { + name: "object has no labels", + o: metav1.ObjectMeta{}, + expected: false, + }, + { + name: "object has no velero backup label", + backupName: "csi-b1", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + expected: false, + }, + { + name: "object has velero backup label but value not equal to backup name", + backupName: "csi-b1", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "velero.io/backup-name": "does-not-match", + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + expected: false, + }, + { + name: "object has backup label with matching backup name value", + backupName: "does-match", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "velero.io/backup-name": "does-match", + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + expected: true, + }, + } + + for _, tc := range testCases { + actual := HasBackupLabel(&tc.o, tc.backupName) + assert.Equal(t, tc.expected, actual) + } +} + +func TestDeleteVolumeSnapshots(t *testing.T) { + tests := []struct { + name string + vs snapshotv1api.VolumeSnapshot + vsc snapshotv1api.VolumeSnapshotContent + expectedVS snapshotv1api.VolumeSnapshot + expectedVSC snapshotv1api.VolumeSnapshotContent + }{ + { + name: "VS is ReadyToUse, and VS has corresponding VSC. VS should be deleted.", + vs: *builder.ForVolumeSnapshot("velero", "vs1").ObjectMeta(builder.WithLabels("testing-vs", "vs1")).Status().BoundVolumeSnapshotContentName("vsc1").Result(), + vsc: *builder.ForVolumeSnapshotContent("vsc1").DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete).Status(&snapshotv1api.VolumeSnapshotContentStatus{}).Result(), + expectedVS: snapshotv1api.VolumeSnapshot{}, + expectedVSC: *builder.ForVolumeSnapshotContent("vsc1").DeletionPolicy(snapshotv1api.VolumeSnapshotContentRetain).VolumeSnapshotRef("ns-", "name-").Result(), + }, + { + name: "VS is ReadyToUse, and VS has corresponding VSC. Concurrent test.", + vs: *builder.ForVolumeSnapshot("velero", "vs1").ObjectMeta(builder.WithLabels("testing-vs", "vs1")).Status().BoundVolumeSnapshotContentName("vsc1").Result(), + vsc: *builder.ForVolumeSnapshotContent("vsc1").DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete).Status(&snapshotv1api.VolumeSnapshotContentStatus{}).Result(), + expectedVS: snapshotv1api.VolumeSnapshot{}, + expectedVSC: *builder.ForVolumeSnapshotContent("vsc1").DeletionPolicy(snapshotv1api.VolumeSnapshotContentRetain).VolumeSnapshotRef("ns-", "name-").Result(), + }, + { + name: "VS status is nil. VSC should not be modified.", + vs: *builder.ForVolumeSnapshot("velero", "vs1").ObjectMeta(builder.WithLabels("testing-vs", "vs1")).Result(), + vsc: *builder.ForVolumeSnapshotContent("vsc1").DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete).Status(&snapshotv1api.VolumeSnapshotContentStatus{}).Result(), + expectedVS: snapshotv1api.VolumeSnapshot{}, + expectedVSC: *builder.ForVolumeSnapshotContent("vsc1").DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete).Result(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + vsClient := snapshotFake.NewSimpleClientset() + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatText) + backup := builder.ForBackup(velerov1api.DefaultNamespace, "backup-1").DefaultVolumesToFsBackup(false).Result() + + _, err := vsClient.SnapshotV1().VolumeSnapshots(tc.vs.Namespace).Create(context.Background(), &tc.vs, metav1.CreateOptions{}) + require.NoError(t, err) + _, err = vsClient.SnapshotV1().VolumeSnapshotContents().Create(context.Background(), &tc.vsc, metav1.CreateOptions{}) + require.NoError(t, err) + + DeleteVolumeSnapshot(tc.vs, tc.vsc, backup, vsClient.SnapshotV1(), logger) + + vsList, err := vsClient.SnapshotV1().VolumeSnapshots("velero").List(context.TODO(), metav1.ListOptions{}) + require.NoError(t, err) + if tc.expectedVS.Name == "" { + require.Equal(t, 0, len(vsList.Items)) + } else { + require.Equal(t, tc.expectedVS.Status, vsList.Items[0].Status) + require.Equal(t, tc.expectedVS.Spec, vsList.Items[0].Spec) + } + + vscList, err := vsClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + require.NoError(t, err) + require.Equal(t, 1, len(vscList.Items)) + require.Equal(t, tc.expectedVSC.Spec, vscList.Items[0].Spec) + }) + } +} diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index afa3f2b0dd8..ecf26878efc 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -57,9 +58,9 @@ func (e *ExcludeFromBackup) Init() error { *e.NSIncluded = append(*e.NSIncluded, createNSName) } e.labels = map[string]string{ - "velero.io/exclude-from-backup": "true", + velerov1api.ExcludeFromBackupLabel: "true", } - e.labelSelector = "velero.io/exclude-from-backup" + e.labelSelector = velerov1api.ExcludeFromBackupLabel e.BackupArgs = []string{ "create", "--namespace", e.VeleroCfg.VeleroNamespace, "backup", e.BackupName, @@ -82,7 +83,7 @@ func (e *ExcludeFromBackup) CreateResources() error { "meaningless-label-resource-to-include": "true", } label2 := map[string]string{ - "velero.io/exclude-from-backup": "false", + velerov1api.ExcludeFromBackupLabel: "false", } fmt.Printf("Creating resources in namespace ...%s\n", namespace) if err := CreateNamespace(e.Ctx, e.Client, namespace); err != nil {