Skip to content

Commit

Permalink
Support snapshot feature in rbdplugin
Browse files Browse the repository at this point in the history
  • Loading branch information
@sngchlko
sngchlko committed Aug 8, 2018
1 parent c93466b commit b1ccdbb
Show file tree
Hide file tree
Showing 3 changed files with 482 additions and 9 deletions.
239 changes: 235 additions & 4 deletions pkg/rbd/controllerserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@ package rbd

import (
"fmt"
"os/exec"
"path"
"syscall"
"time"

"github.com/container-storage-interface/spec/lib/go/csi/v0"
"github.com/golang/glog"
Expand Down Expand Up @@ -77,7 +80,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
return nil, err
}

// Generating Volume Name and Volume ID, as accoeding to CSI spec they MUST be different
// Generating Volume Name and Volume ID, as according to CSI spec they MUST be different
volName := req.GetName()
uniqueID := uuid.NewUUID().String()
if len(volName) == 0 {
Expand All @@ -97,14 +100,39 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
// Check if there is already RBD image with requested name
found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets())
if !found {
if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil {
// if VolumeContentSource is not nil, this request is for snapshot
if req.VolumeContentSource != nil {
snapshot := req.VolumeContentSource.GetSnapshot()
if snapshot == nil {
return nil, status.Error(codes.InvalidArgument, "Volume Snapshot cannot be empty")
}

snapshotID := snapshot.GetId()
if snapshotID == "" {
return nil, status.Error(codes.InvalidArgument, "Volume Snapshot ID cannot be empty")
}

rbdSnap := &rbdSnapshot{}
if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
return nil, err
}

err = restoreSnapshot(rbdVol, rbdSnap, req.GetControllerCreateSecrets())
if err != nil {
glog.Warningf("failed to create volume: %v", err)
return nil, err
}
glog.V(4).Infof("create volume %s from snapshot %s", volName, rbdSnap.SnapName)
} else {
if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil {
if err != nil {
glog.Warningf("failed to create volume: %v", err)
return nil, err
}
}
glog.V(4).Infof("create volume %s", volName)
}
glog.V(4).Infof("create volume %s", volName)
}

// Storing volInfo into a persistent file.
if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil {
glog.Warningf("rbd: failed to store volInfo with error: %v", err)
Expand Down Expand Up @@ -162,3 +190,206 @@ func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *
func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
return &csi.ControllerPublishVolumeResponse{}, nil
}

func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil {
glog.Warningf("invalid create snapshot req: %v", req)
return nil, err
}

// Check sanity of request Snapshot Name, Source Volume Id
if len(req.Name) == 0 {
return nil, status.Error(codes.InvalidArgument, "Snapshot Name cannot be empty")
}
if len(req.SourceVolumeId) == 0 {
return nil, status.Error(codes.InvalidArgument, "Source Volume ID cannot be empty")
}

// Need to check for already existing snapshot name, and if found
// check for the requested source volume id and already allocated source volume id
if exSnap, err := getRBDSnapshotByName(req.GetName()); err == nil {
if req.SourceVolumeId == exSnap.SourceVolumeID {
return &csi.CreateSnapshotResponse{
Snapshot: &csi.Snapshot{
SizeBytes: exSnap.SizeBytes,
Id: exSnap.SnapID,
SourceVolumeId: exSnap.SourceVolumeID,
CreatedAt: exSnap.CreatedAt,
Status: &csi.SnapshotStatus{
Type: csi.SnapshotStatus_READY,
},
},
}, nil
}
return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Snapshot with the same name: %s but with different source volume id already exist", req.GetName()))
}

rbdSnap, err := getRBDSnapshotOptions(req.GetParameters())
if err != nil {
return nil, err
}

// Generating Snapshot Name and Snapshot ID, as according to CSI spec they MUST be different
snapName := req.GetName()
uniqueID := uuid.NewUUID().String()
rbdVolume, err := getRBDVolumeByID(req.GetSourceVolumeId())
if err != nil {
return nil, status.Error(codes.NotFound, fmt.Sprintf("Source Volume ID %s cannot found", req.GetSourceVolumeId()))
}
rbdSnap.VolName = rbdVolume.VolName
rbdSnap.SnapName = snapName
snapshotID := "csi-rbd-snapshot-" + uniqueID
rbdSnap.SnapID = snapshotID
rbdSnap.SourceVolumeID = req.GetSourceVolumeId()
rbdSnap.SizeBytes = rbdVolume.VolSize

err = createSnapshot(rbdSnap, req.GetCreateSnapshotSecrets())
// if we already have the snapshot, return the snapshot
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
if status.ExitStatus() == int(syscall.EEXIST) {
glog.Warningf("Snapshot with the same name: %s, we return this.", req.GetName())
} else {
glog.Warningf("failed to create snapshot: %v", err)
return nil, err
}
} else {
glog.Warningf("failed to create snapshot: %v", err)
return nil, err
}
} else {
glog.Warningf("failed to create snapshot: %v", err)
return nil, err
}
} else {
glog.V(4).Infof("create snapshot %s", snapName)
err = protectSnapshot(rbdSnap, req.GetCreateSnapshotSecrets())

if err != nil {
err = deleteSnapshot(rbdSnap, req.GetCreateSnapshotSecrets())
if err != nil {
return nil, fmt.Errorf("snapshot is created but failed to protect and delete snapshot: %v", err)
}
return nil, fmt.Errorf("Snapshot is created but failed to protect snapshot")
}
}

rbdSnap.CreatedAt = time.Now().UnixNano()

// Storing snapInfo into a persistent file.
if err := persistSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
glog.Warningf("rbd: failed to store sanpInfo with error: %v", err)
return nil, err
}
rbdSnapshots[snapshotID] = *rbdSnap
return &csi.CreateSnapshotResponse{
Snapshot: &csi.Snapshot{
SizeBytes: rbdSnap.SizeBytes,
Id: snapshotID,
SourceVolumeId: req.GetSourceVolumeId(),
CreatedAt: rbdSnap.CreatedAt,
Status: &csi.SnapshotStatus{
Type: csi.SnapshotStatus_READY,
},
},
}, nil
}

func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil {
glog.Warningf("invalid delete snapshot req: %v", req)
return nil, err
}

snapshotID := req.GetSnapshotId()
if snapshotID == "" {
return nil, status.Error(codes.InvalidArgument, "Snapshot ID cannot be empty")
}
rbdSnap := &rbdSnapshot{}
if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
return nil, err
}

// Unprotect snapshot
err := unprotectSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets())
if err != nil {
return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
}

// Deleting snapshot
glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName)
if err := deleteSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()); err != nil {
return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
}

// Removing persistent storage file for the unmapped snapshot
if err := deleteSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap")); err != nil {
return nil, err
}

delete(rbdSnapshots, snapshotID)

return &csi.DeleteSnapshotResponse{}, nil
}

func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS); err != nil {
glog.Warningf("invalid list snapshot req: %v", req)
return nil, err
}

sourceVolumeId := req.GetSourceVolumeId()

// TODO (sngchlko) list with token

// list only a specific snapshot which has snapshot ID
if snapshotID := req.GetSnapshotId(); snapshotID != "" {
if rbdSnap, ok := rbdSnapshots[snapshotID]; ok {
// if source volume ID also set, check source volume id on the cache.
if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId {
return nil, status.Error(codes.Unknown, fmt.Sprintf("Requested Source Volume ID %s is different from %s", sourceVolumeId, rbdSnap.SourceVolumeID))
}
return &csi.ListSnapshotsResponse{
Entries: []*csi.ListSnapshotsResponse_Entry{
{
Snapshot: &csi.Snapshot{
SizeBytes: rbdSnap.SizeBytes,
Id: rbdSnap.SnapID,
SourceVolumeId: rbdSnap.SourceVolumeID,
CreatedAt: rbdSnap.CreatedAt,
Status: &csi.SnapshotStatus{
Type: csi.SnapshotStatus_READY,
},
},
},
},
}, nil
} else {
return nil, status.Error(codes.NotFound, fmt.Sprintf("Snapshot ID %s cannot found", snapshotID))
}
}

entries := []*csi.ListSnapshotsResponse_Entry{}
for _, rbdSnap := range rbdSnapshots {
// if source volume ID also set, check source volume id on the cache.
if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId {
continue
}
entries = append(entries, &csi.ListSnapshotsResponse_Entry{
Snapshot: &csi.Snapshot{
SizeBytes: rbdSnap.SizeBytes,
Id: rbdSnap.SnapID,
SourceVolumeId: rbdSnap.SourceVolumeID,
CreatedAt: rbdSnap.CreatedAt,
Status: &csi.SnapshotStatus{
Type: csi.SnapshotStatus_READY,
},
},
})
}

return &csi.ListSnapshotsResponse{
Entries: entries,
}, nil
}
53 changes: 48 additions & 5 deletions pkg/rbd/rbd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,25 +51,66 @@ type rbd struct {

var (
rbdDriver *rbd
version = "0.2.0"
version = "0.3.0"
)

var rbdVolumes map[string]rbdVolume
var rbdSnapshots map[string]rbdSnapshot

// Init checks for the persistent volume file and loads all found volumes
// into a memory structure
func init() {
rbdVolumes = map[string]rbdVolume{}
rbdSnapshots = map[string]rbdSnapshot{}
if _, err := os.Stat(path.Join(PluginFolder, "controller")); os.IsNotExist(err) {
glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller"))
if err := os.Mkdir(path.Join(PluginFolder, "controller"), 0755); err != nil {
glog.Fatalf("Failed to create a controller's volumes folder with error: %v\n", err)
}
} else {
// Since "controller" folder exists, it means the rbdplugin has already been running, it means
// there might be some volumes left, they must be re-inserted into rbdVolumes map
loadExVolumes()
}
if _, err := os.Stat(path.Join(PluginFolder, "controller-snap")); os.IsNotExist(err) {
glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller-snap"))
if err := os.Mkdir(path.Join(PluginFolder, "controller-snap"), 0755); err != nil {
glog.Fatalf("Failed to create a controller's snapshots folder with error: %v\n", err)
}
} else {
// Since "controller-snap" folder exists, it means the rbdplugin has already been running, it means
// there might be some snapshots left, they must be re-inserted into rbdSnapshots map
loadExSnapshots()
}
}

// loadExSnapshots check for any *.json files in the PluginFolder/controller-snap folder
// and loads then into rbdSnapshots map
func loadExSnapshots() {
rbdSnap := rbdSnapshot{}
files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller-snap"))
if err != nil {
glog.Infof("rbd: failed to read controller's snapshots folder: %s error:%v", path.Join(PluginFolder, "controller-snap"), err)
return
}
// Since "controller" folder exists, it means the rbdplugin has already been running, it means
// there might be some volumes left, they must be re-inserted into rbdVolumes map
loadExVolumes()
for _, f := range files {
if !strings.HasSuffix(f.Name(), ".json") {
continue
}
fp, err := os.Open(path.Join(PluginFolder, "controller-snap", f.Name()))
if err != nil {
glog.Infof("rbd: open file: %s err %%v", f.Name(), err)
continue
}
decoder := json.NewDecoder(fp)
if err = decoder.Decode(&rbdSnap); err != nil {
glog.Infof("rbd: decode file: %s err: %v", f.Name(), err)
fp.Close()
continue
}
rbdSnapshots[rbdSnap.SnapID] = rbdSnap
}
glog.Infof("rbd: Loaded %d snapshots from %s", len(rbdSnapshots), path.Join(PluginFolder, "controller-snap"))
}

// loadExVolumes check for any *.json files in the PluginFolder/controller folder
Expand All @@ -78,7 +119,7 @@ func loadExVolumes() {
rbdVol := rbdVolume{}
files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller"))
if err != nil {
glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err)
glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err)
return
}
for _, f := range files {
Expand Down Expand Up @@ -134,6 +175,8 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string) {
rbd.driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
})
rbd.driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER})

Expand Down
Loading

0 comments on commit b1ccdbb

Please sign in to comment.