Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for standalone snapshot creation #467

Closed
wants to merge 1 commit into from
Closed

Add support for standalone snapshot creation #467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
# need for docker build
sudo: true
dist: xenial
dist: bionic

addons:
apt:
Expand Down
5 changes: 3 additions & 2 deletions cmd/cephcsi.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ var (
metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")

// rbd related flags
containerized = flag.Bool("containerized", true, "whether run as containerized")
containerized = flag.Bool("containerized", true, "whether run as containerized")
rbdMaxCloneDepth = flag.Uint("rbdmaximumclonedepth", 16, "Maximum number of nested volume clones that are taken before a flatten occurs")
Copy link

@dillaman dillaman Jul 29, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: while 16 is technically the maximum, I think it should be set much lower before a flatten is required since a deeper depth will impact IO speeds. For example, Cinder uses a max depth of 5 [1] so perhaps copy that as a starting point.

Additionally, in the future it might be nice to have a soft vs hard depth limit (i.e. when you hit the soft-depth, it doesn't block PV/snapshot creation but it kicks off a background ceph rbd task add flatten task whereas the hard limit does block until the flatten is complete). In that case, can we name this something like rbdhardmaxclonedepth)?

[1] https://github.com/openstack/cinder/blob/master/cinder/volume/drivers/rbd.py#L83


// cephfs related flags
volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
Expand Down Expand Up @@ -117,7 +118,7 @@ func main() {
}
}
driver := rbd.NewDriver()
driver.Run(dname, *nodeID, *endpoint, *instanceID, *containerized, cp)
driver.Run(dname, *nodeID, *endpoint, *instanceID, *containerized, *rbdMaxCloneDepth, cp)

case cephfsType:
cephfs.PluginFolder += dname
Expand Down
1 change: 1 addition & 0 deletions deploy/rbd/helm/templates/provisioner-statefulset.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ spec:
- "--v=5"
- "--drivername=$(DRIVER_NAME)"
- "--containerized=true"
- "--rbdmaximumclonedepth=14"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: leave it at its default and don't override?

env:
- name: HOST_ROOTFS
value: "/rootfs"
Expand Down
1 change: 1 addition & 0 deletions deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ spec:
- "--v=5"
- "--drivername=rbd.csi.ceph.com"
- "--containerized=true"
- "--rbdmaximumclonedepth=14"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: same comment here

env:
- name: HOST_ROOTFS
value: "/rootfs"
Expand Down
36 changes: 2 additions & 34 deletions e2e/cephfs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package e2e

import (
"fmt"
"time"

. "github.com/onsi/ginkgo" // nolint
Expand Down Expand Up @@ -74,46 +73,15 @@ var _ = Describe("cephfs", func() {
By("create and delete a PVC", func() {
By("create a PVC and Bind it to an app", func() {
validatePVCAndAppBinding(pvcPath, appPath, f)

})

By("create a PVC and Bind it to an app with normal user", func() {
validateNormalUserPVCAccess(pvcPath, f)
})

By("create/delete multiple PVCs and Apps", func() {
totalCount := 2
pvc, err := loadPVC(pvcPath)
if err != nil {
Fail(err.Error())
}
pvc.Namespace = f.UniqueName

app, err := loadApp(appPath)
if err != nil {
Fail(err.Error())
}
app.Namespace = f.UniqueName
// create pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := createPVCAndApp(name, f, pvc, app)
if err != nil {
Fail(err.Error())
}

}
// TODO add cephfs backend validation

// delete pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := deletePVCAndApp(name, f, pvc, app)
if err != nil {
Fail(err.Error())
}

}
totalCount := 10
validatePVCAndApp(false, pvcPath, appPath, totalCount, f)
})

})
Expand Down
102 changes: 8 additions & 94 deletions e2e/rbd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package e2e

import (
"fmt"
"time"

. "github.com/onsi/ginkgo" // nolint
Expand Down Expand Up @@ -68,6 +67,8 @@ var _ = Describe("RBD", func() {
appClonePath := rbdExamplePath + "pod-restore.yaml"
snapshotPath := rbdExamplePath + "snapshot.yaml"

totalCount := 20

By("checking provisioner statefulset is running")
timeout := time.Duration(deployTimeout) * time.Minute
err := framework.WaitForStatefulSetReplicasReady(rbdDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout)
Expand All @@ -89,105 +90,18 @@ var _ = Describe("RBD", func() {
validateNormalUserPVCAccess(pvcPath, f)
})

By("create a PVC clone and Bind it to an app", func() {
createRBDSnapshotClass(f)
pvc, err := loadPVC(pvcPath)
if err != nil {
Fail(err.Error())
}

pvc.Namespace = f.UniqueName
e2elog.Logf("The PVC template %+v", pvc)
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
Fail(err.Error())
}
// validate created backend rbd images
images := listRBDImages(f)
if len(images) != 1 {
e2elog.Logf("backend image count %d expected image count %d", len(images), 1)
Fail("validate backend image failed")
}
snap := getSnapshot(snapshotPath)
snap.Namespace = f.UniqueName
snap.Spec.Source.Name = pvc.Name
snap.Spec.Source.Kind = "PersistentVolumeClaim"
err = createSnapshot(&snap, deployTimeout)
if err != nil {
Fail(err.Error())
}
pool := "replicapool"
snapList, err := listSnapshots(f, pool, images[0])
if err != nil {
Fail(err.Error())
}
if len(snapList) != 1 {
e2elog.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1)
Fail("validate backend snapshot failed")
}

validatePVCAndAppBinding(pvcClonePath, appClonePath, f)

err = deleteSnapshot(&snap, deployTimeout)
if err != nil {
Fail(err.Error())
}
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
Fail(err.Error())
}
})

// skipped raw pvc test in travis
// By("create a block type PVC and Bind it to an app", func() {
// validatePVCAndAppBinding(rawPvcPath, rawAppPath, f)
// })

By("create/delete multiple PVCs and Apps", func() {
totalCount := 2
pvc, err := loadPVC(pvcPath)
if err != nil {
Fail(err.Error())
}
pvc.Namespace = f.UniqueName

app, err := loadApp(appPath)
if err != nil {
Fail(err.Error())
}
app.Namespace = f.UniqueName
// create pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := createPVCAndApp(name, f, pvc, app)
if err != nil {
Fail(err.Error())
}

}
// validate created backend rbd images
images := listRBDImages(f)
if len(images) != totalCount {
e2elog.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount)
Fail("validate multiple pvc failed")
}

// delete pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := deletePVCAndApp(name, f, pvc, app)
if err != nil {
Fail(err.Error())
}

}

// validate created backend rbd images
images = listRBDImages(f)
if len(images) > 0 {
e2elog.Logf("left out rbd backend images count %d", len(images))
Fail("validate multiple pvc failed")
}
validatePVCAndApp(true, pvcPath, appPath, totalCount, f)
})

By("create/delete multiple clone PVCs with datasource=snapsphot and Apps", func() {
createRBDSnapshotClass(f)
validateCloneFromSnapshot(pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, totalCount, f)
})
})
})
Expand Down
Loading