Add support for standalone snapshot creation

with the current implementation in ceph-csi, it's not possible to delete the cloned volume if the snapshot is present due to the child linking, to remove this dependency we had a discussion and come up with an idea to separate out the clone and snapshot, so that we can delete the snapshot and cloned image in any order. the steps followed to create an independent snapshot as follows * Create a temporary snapshot from the parent volume * Clone a new image from a temporary snapshot with options `--rbd-default-clone-format 2 --image-feature layering,deep-flatten` * Deletetemprary snapshot created * Create a snapshot with requested Name * Clone a new image from the snapshot with user-provided options * Check the depth of the image as the maximum number of nested volume clones can be (max 16 can be changed based on the configuration) if the depth is reached flatten the newly cloned image * Delete the cloned image (earlier we were removing the image with `rbd rm` command with the new design we will be moving the images to the trash) same applies for normal volume deletion also * Delete the temporary cloned image which was created for a snapshot * Delete the snapshot example commands:- ``` 1) rbd snap create <RBD image for src k8s volume>@<random snap name> 2) rbd clone --rbd-default-clone-format 2 --image-feature layering,deep-flatten <RBD image for src k8s volume>@<random snap <RBD image for temporary snap image> 3) rbd snap rm <RBD image for src k8s volume>@<random snap name> 4) rbd snap create <RBD image for temporary snap image>@<random snap name> <k8s snap name> 5) rbd clone --rbd-default-clone-format 2 --image-feature <k8s dst vol config> <RBD image for temporary snap image>@<random snap name> <RBD image for k8s dst vol> ``` Signed-off-by: Madhu Rajanna <[email protected]>
ceph · Jul 12, 2019 · 444d46d · 444d46d
1 parent 0d566ee
commit 444d46d
Show file tree

Hide file tree

Showing 12 changed files with 555 additions and 260 deletions.
diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go
@@ -47,7 +47,8 @@ var (
 	metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
 
 	// rbd related flags
-	containerized = flag.Bool("containerized", true, "whether run as containerized")
+	containerized    = flag.Bool("containerized", true, "whether run as containerized")
+	rbdMaxCloneDepth = flag.Uint("rbdmaximumclonedepth", 16, "Maximum number of nested volume clones that are taken before a flatten occurs")
 
 	// cephfs related flags
 	volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
@@ -117,7 +118,7 @@ func main() {
 			}
 		}
 		driver := rbd.NewDriver()
-		driver.Run(dname, *nodeID, *endpoint, *instanceID, *containerized, cp)
+		driver.Run(dname, *nodeID, *endpoint, *instanceID, *containerized, *rbdMaxCloneDepth, cp)
 
 	case cephfsType:
 		cephfs.PluginFolder += dname

diff --git a/deploy/rbd/helm/templates/provisioner-statefulset.yaml b/deploy/rbd/helm/templates/provisioner-statefulset.yaml
@@ -88,6 +88,7 @@ spec:
             - "--v=5"
             - "--drivername=$(DRIVER_NAME)"
             - "--containerized=true"
+            - "--rbdmaximumclonedepth=14"
           env:
             - name: HOST_ROOTFS
               value: "/rootfs"

diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
@@ -85,6 +85,7 @@ spec:
             - "--v=5"
             - "--drivername=rbd.csi.ceph.com"
             - "--containerized=true"
+            - "--rbdmaximumclonedepth=14"
           env:
             - name: HOST_ROOTFS
               value: "/rootfs"

diff --git a/e2e/cephfs.go b/e2e/cephfs.go
@@ -1,7 +1,6 @@
 package e2e
 
 import (
-	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo" // nolint
@@ -74,46 +73,15 @@ var _ = Describe("cephfs", func() {
 			By("create and delete a PVC", func() {
 				By("create a PVC and Bind it to an app", func() {
 					validatePVCAndAppBinding(pvcPath, appPath, f)
-
 				})
 
 				By("create a PVC and Bind it to an app with normal user", func() {
 					validateNormalUserPVCAccess(pvcPath, f)
 				})
 
 				By("create/delete multiple PVCs and Apps", func() {
-					totalCount := 2
-					pvc, err := loadPVC(pvcPath)
-					if err != nil {
-						Fail(err.Error())
-					}
-					pvc.Namespace = f.UniqueName
-
-					app, err := loadApp(appPath)
-					if err != nil {
-						Fail(err.Error())
-					}
-					app.Namespace = f.UniqueName
-					// create pvc and app
-					for i := 0; i < totalCount; i++ {
-						name := fmt.Sprintf("%s%d", f.UniqueName, i)
-						err := createPVCAndApp(name, f, pvc, app)
-						if err != nil {
-							Fail(err.Error())
-						}
-
-					}
-					// TODO add cephfs backend validation
-
-					// delete pvc and app
-					for i := 0; i < totalCount; i++ {
-						name := fmt.Sprintf("%s%d", f.UniqueName, i)
-						err := deletePVCAndApp(name, f, pvc, app)
-						if err != nil {
-							Fail(err.Error())
-						}
-
-					}
+					totalCount := 10
+					validatePVCAndApp(false, pvcPath, appPath, totalCount, f)
 				})
 
 			})

diff --git a/e2e/rbd.go b/e2e/rbd.go
@@ -1,7 +1,6 @@
 package e2e
 
 import (
-	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo" // nolint
@@ -68,6 +67,8 @@ var _ = Describe("RBD", func() {
 			appClonePath := rbdExamplePath + "pod-restore.yaml"
 			snapshotPath := rbdExamplePath + "snapshot.yaml"
 
+			totalCount := 20
+
 			By("checking provisioner statefulset is running")
 			timeout := time.Duration(deployTimeout) * time.Minute
 			err := framework.WaitForStatefulSetReplicasReady(rbdDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout)
@@ -89,105 +90,18 @@ var _ = Describe("RBD", func() {
 				validateNormalUserPVCAccess(pvcPath, f)
 			})
 
-			By("create a PVC clone and Bind it to an app", func() {
-				createRBDSnapshotClass(f)
-				pvc, err := loadPVC(pvcPath)
-				if err != nil {
-					Fail(err.Error())
-				}
-
-				pvc.Namespace = f.UniqueName
-				e2elog.Logf("The PVC  template %+v", pvc)
-				err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
-				if err != nil {
-					Fail(err.Error())
-				}
-				// validate created backend rbd images
-				images := listRBDImages(f)
-				if len(images) != 1 {
-					e2elog.Logf("backend image count %d expected image count %d", len(images), 1)
-					Fail("validate backend image failed")
-				}
-				snap := getSnapshot(snapshotPath)
-				snap.Namespace = f.UniqueName
-				snap.Spec.Source.Name = pvc.Name
-				snap.Spec.Source.Kind = "PersistentVolumeClaim"
-				err = createSnapshot(&snap, deployTimeout)
-				if err != nil {
-					Fail(err.Error())
-				}
-				pool := "replicapool"
-				snapList, err := listSnapshots(f, pool, images[0])
-				if err != nil {
-					Fail(err.Error())
-				}
-				if len(snapList) != 1 {
-					e2elog.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1)
-					Fail("validate backend snapshot failed")
-				}
-
-				validatePVCAndAppBinding(pvcClonePath, appClonePath, f)
-
-				err = deleteSnapshot(&snap, deployTimeout)
-				if err != nil {
-					Fail(err.Error())
-				}
-				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
-				if err != nil {
-					Fail(err.Error())
-				}
-			})
-
 			// skipped raw pvc test in travis
 			// By("create a block type PVC and Bind it to an app", func() {
 			// 	validatePVCAndAppBinding(rawPvcPath, rawAppPath, f)
 			// })
 
 			By("create/delete multiple PVCs and Apps", func() {
-				totalCount := 2
-				pvc, err := loadPVC(pvcPath)
-				if err != nil {
-					Fail(err.Error())
-				}
-				pvc.Namespace = f.UniqueName
-
-				app, err := loadApp(appPath)
-				if err != nil {
-					Fail(err.Error())
-				}
-				app.Namespace = f.UniqueName
-				// create pvc and app
-				for i := 0; i < totalCount; i++ {
-					name := fmt.Sprintf("%s%d", f.UniqueName, i)
-					err := createPVCAndApp(name, f, pvc, app)
-					if err != nil {
-						Fail(err.Error())
-					}
-
-				}
-				// validate created backend rbd images
-				images := listRBDImages(f)
-				if len(images) != totalCount {
-					e2elog.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount)
-					Fail("validate multiple pvc failed")
-				}
-
-				// delete pvc and app
-				for i := 0; i < totalCount; i++ {
-					name := fmt.Sprintf("%s%d", f.UniqueName, i)
-					err := deletePVCAndApp(name, f, pvc, app)
-					if err != nil {
-						Fail(err.Error())
-					}
-
-				}
-
-				// validate created backend rbd images
-				images = listRBDImages(f)
-				if len(images) > 0 {
-					e2elog.Logf("left out rbd backend images count %d", len(images))
-					Fail("validate multiple pvc failed")
-				}
+				validatePVCAndApp(true, pvcPath, appPath, totalCount, f)
+			})
+
+			By("create/delete multiple clone PVCs with datasource=snapsphot and Apps", func() {
+				createRBDSnapshotClass(f)
+				validateCloneFromSnapshot(pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, totalCount, f)
 			})
 		})
 	})