diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 4c986e776d5..546a27d90e4 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -18,7 +18,10 @@ package rbd import ( "fmt" + "os/exec" "path" + "syscall" + "time" "github.com/container-storage-interface/spec/lib/go/csi/v0" "github.com/golang/glog" @@ -77,7 +80,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } - // Generating Volume Name and Volume ID, as accoeding to CSI spec they MUST be different + // Generating Volume Name and Volume ID, as according to CSI spec they MUST be different volName := req.GetName() uniqueID := uuid.NewUUID().String() if len(volName) == 0 { @@ -97,14 +100,39 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol // Check if there is already RBD image with requested name found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets()) if !found { - if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil { + // if VolumeContentSource is not nil, this request is for snapshot + if req.VolumeContentSource != nil { + snapshot := req.VolumeContentSource.GetSnapshot() + if snapshot == nil { + return nil, status.Error(codes.InvalidArgument, "Volume Snapshot cannot be empty") + } + + snapshotID := snapshot.GetId() + if snapshotID == "" { + return nil, status.Error(codes.InvalidArgument, "Volume Snapshot ID cannot be empty") + } + + rbdSnap := &rbdSnapshot{} + if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil { + return nil, err + } + + err = restoreSnapshot(rbdVol, rbdSnap, req.GetControllerCreateSecrets()) if err != nil { - glog.Warningf("failed to create volume: %v", err) return nil, err } + glog.V(4).Infof("create volume %s from snapshot %s", volName, rbdSnap.SnapName) + } else { + if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil { + if err != nil { + glog.Warningf("failed to create volume: %v", err) + return nil, err + } + } + glog.V(4).Infof("create volume %s", volName) } - glog.V(4).Infof("create volume %s", volName) } + // Storing volInfo into a persistent file. if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil { glog.Warningf("rbd: failed to store volInfo with error: %v", err) @@ -162,3 +190,206 @@ func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req * func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { return &csi.ControllerPublishVolumeResponse{}, nil } + +func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil { + glog.Warningf("invalid create snapshot req: %v", req) + return nil, err + } + + // Check sanity of request Snapshot Name, Source Volume Id + if len(req.Name) == 0 { + return nil, status.Error(codes.InvalidArgument, "Snapshot Name cannot be empty") + } + if len(req.SourceVolumeId) == 0 { + return nil, status.Error(codes.InvalidArgument, "Source Volume ID cannot be empty") + } + + // Need to check for already existing snapshot name, and if found + // check for the requested source volume id and already allocated source volume id + if exSnap, err := getRBDSnapshotByName(req.GetName()); err == nil { + if req.SourceVolumeId == exSnap.SourceVolumeID { + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SizeBytes: exSnap.SizeBytes, + Id: exSnap.SnapID, + SourceVolumeId: exSnap.SourceVolumeID, + CreatedAt: exSnap.CreatedAt, + Status: &csi.SnapshotStatus{ + Type: csi.SnapshotStatus_READY, + }, + }, + }, nil + } + return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Snapshot with the same name: %s but with different source volume id already exist", req.GetName())) + } + + rbdSnap, err := getRBDSnapshotOptions(req.GetParameters()) + if err != nil { + return nil, err + } + + // Generating Snapshot Name and Snapshot ID, as according to CSI spec they MUST be different + snapName := req.GetName() + uniqueID := uuid.NewUUID().String() + rbdVolume, err := getRBDVolumeByID(req.GetSourceVolumeId()) + if err != nil { + return nil, status.Error(codes.NotFound, fmt.Sprintf("Source Volume ID %s cannot found", req.GetSourceVolumeId())) + } + rbdSnap.VolName = rbdVolume.VolName + rbdSnap.SnapName = snapName + snapshotID := "csi-rbd-snapshot-" + uniqueID + rbdSnap.SnapID = snapshotID + rbdSnap.SourceVolumeID = req.GetSourceVolumeId() + rbdSnap.SizeBytes = rbdVolume.VolSize + + err = createSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + // if we already have the snapshot, return the snapshot + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { + if status.ExitStatus() == int(syscall.EEXIST) { + glog.Warningf("Snapshot with the same name: %s, we return this.", req.GetName()) + } else { + glog.Warningf("failed to create snapshot: %v", err) + return nil, err + } + } else { + glog.Warningf("failed to create snapshot: %v", err) + return nil, err + } + } else { + glog.Warningf("failed to create snapshot: %v", err) + return nil, err + } + } else { + glog.V(4).Infof("create snapshot %s", snapName) + err = protectSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + + if err != nil { + err = deleteSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + if err != nil { + return nil, fmt.Errorf("snapshot is created but failed to protect and delete snapshot: %v", err) + } + return nil, fmt.Errorf("Snapshot is created but failed to protect snapshot") + } + } + + rbdSnap.CreatedAt = time.Now().UnixNano() + + // Storing snapInfo into a persistent file. + if err := persistSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil { + glog.Warningf("rbd: failed to store sanpInfo with error: %v", err) + return nil, err + } + rbdSnapshots[snapshotID] = *rbdSnap + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SizeBytes: rbdSnap.SizeBytes, + Id: snapshotID, + SourceVolumeId: req.GetSourceVolumeId(), + CreatedAt: rbdSnap.CreatedAt, + Status: &csi.SnapshotStatus{ + Type: csi.SnapshotStatus_READY, + }, + }, + }, nil +} + +func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil { + glog.Warningf("invalid delete snapshot req: %v", req) + return nil, err + } + + snapshotID := req.GetSnapshotId() + if snapshotID == "" { + return nil, status.Error(codes.InvalidArgument, "Snapshot ID cannot be empty") + } + rbdSnap := &rbdSnapshot{} + if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil { + return nil, err + } + + // Unprotect snapshot + err := unprotectSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()) + if err != nil { + return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) + } + + // Deleting snapshot + glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName) + if err := deleteSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()); err != nil { + return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) + } + + // Removing persistent storage file for the unmapped snapshot + if err := deleteSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap")); err != nil { + return nil, err + } + + delete(rbdSnapshots, snapshotID) + + return &csi.DeleteSnapshotResponse{}, nil +} + +func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS); err != nil { + glog.Warningf("invalid list snapshot req: %v", req) + return nil, err + } + + sourceVolumeId := req.GetSourceVolumeId() + + // TODO (sngchlko) list with token + + // list only a specific snapshot which has snapshot ID + if snapshotID := req.GetSnapshotId(); snapshotID != "" { + if rbdSnap, ok := rbdSnapshots[snapshotID]; ok { + // if source volume ID also set, check source volume id on the cache. + if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId { + return nil, status.Error(codes.Unknown, fmt.Sprintf("Requested Source Volume ID %s is different from %s", sourceVolumeId, rbdSnap.SourceVolumeID)) + } + return &csi.ListSnapshotsResponse{ + Entries: []*csi.ListSnapshotsResponse_Entry{ + { + Snapshot: &csi.Snapshot{ + SizeBytes: rbdSnap.SizeBytes, + Id: rbdSnap.SnapID, + SourceVolumeId: rbdSnap.SourceVolumeID, + CreatedAt: rbdSnap.CreatedAt, + Status: &csi.SnapshotStatus{ + Type: csi.SnapshotStatus_READY, + }, + }, + }, + }, + }, nil + } else { + return nil, status.Error(codes.NotFound, fmt.Sprintf("Snapshot ID %s cannot found", snapshotID)) + } + } + + entries := []*csi.ListSnapshotsResponse_Entry{} + for _, rbdSnap := range rbdSnapshots { + // if source volume ID also set, check source volume id on the cache. + if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId { + continue + } + entries = append(entries, &csi.ListSnapshotsResponse_Entry{ + Snapshot: &csi.Snapshot{ + SizeBytes: rbdSnap.SizeBytes, + Id: rbdSnap.SnapID, + SourceVolumeId: rbdSnap.SourceVolumeID, + CreatedAt: rbdSnap.CreatedAt, + Status: &csi.SnapshotStatus{ + Type: csi.SnapshotStatus_READY, + }, + }, + }) + } + + return &csi.ListSnapshotsResponse{ + Entries: entries, + }, nil +} diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index 569e97219f4..09395a286a9 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -51,25 +51,66 @@ type rbd struct { var ( rbdDriver *rbd - version = "0.2.0" + version = "0.3.0" ) var rbdVolumes map[string]rbdVolume +var rbdSnapshots map[string]rbdSnapshot // Init checks for the persistent volume file and loads all found volumes // into a memory structure func init() { rbdVolumes = map[string]rbdVolume{} + rbdSnapshots = map[string]rbdSnapshot{} if _, err := os.Stat(path.Join(PluginFolder, "controller")); os.IsNotExist(err) { glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller")) if err := os.Mkdir(path.Join(PluginFolder, "controller"), 0755); err != nil { glog.Fatalf("Failed to create a controller's volumes folder with error: %v\n", err) } + } else { + // Since "controller" folder exists, it means the rbdplugin has already been running, it means + // there might be some volumes left, they must be re-inserted into rbdVolumes map + loadExVolumes() + } + if _, err := os.Stat(path.Join(PluginFolder, "controller-snap")); os.IsNotExist(err) { + glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller-snap")) + if err := os.Mkdir(path.Join(PluginFolder, "controller-snap"), 0755); err != nil { + glog.Fatalf("Failed to create a controller's snapshots folder with error: %v\n", err) + } + } else { + // Since "controller-snap" folder exists, it means the rbdplugin has already been running, it means + // there might be some snapshots left, they must be re-inserted into rbdSnapshots map + loadExSnapshots() + } +} + +// loadExSnapshots check for any *.json files in the PluginFolder/controller-snap folder +// and loads then into rbdSnapshots map +func loadExSnapshots() { + rbdSnap := rbdSnapshot{} + files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller-snap")) + if err != nil { + glog.Infof("rbd: failed to read controller's snapshots folder: %s error:%v", path.Join(PluginFolder, "controller-snap"), err) return } - // Since "controller" folder exists, it means the rbdplugin has already been running, it means - // there might be some volumes left, they must be re-inserted into rbdVolumes map - loadExVolumes() + for _, f := range files { + if !strings.HasSuffix(f.Name(), ".json") { + continue + } + fp, err := os.Open(path.Join(PluginFolder, "controller-snap", f.Name())) + if err != nil { + glog.Infof("rbd: open file: %s err %%v", f.Name(), err) + continue + } + decoder := json.NewDecoder(fp) + if err = decoder.Decode(&rbdSnap); err != nil { + glog.Infof("rbd: decode file: %s err: %v", f.Name(), err) + fp.Close() + continue + } + rbdSnapshots[rbdSnap.SnapID] = rbdSnap + } + glog.Infof("rbd: Loaded %d snapshots from %s", len(rbdSnapshots), path.Join(PluginFolder, "controller-snap")) } // loadExVolumes check for any *.json files in the PluginFolder/controller folder @@ -78,7 +119,7 @@ func loadExVolumes() { rbdVol := rbdVolume{} files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller")) if err != nil { - glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err) + glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err) return } for _, f := range files { @@ -134,6 +175,8 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string) { rbd.driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, + csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, }) rbd.driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}) diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index afdcd21e585..796ed606a30 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -56,6 +56,17 @@ type rbdVolume struct { VolSize int64 `json:"volSize"` } +type rbdSnapshot struct { + SourceVolumeID string `json:"sourceVolumeID"` + VolName string `json:"volName"` + SnapName string `json:"snapName"` + SnapID string `json:"sanpID"` + Monitors string `json:"monitors"` + Pool string `json:"pool"` + CreatedAt int64 `json:"createdAt"` + SizeBytes int64 `json:"sizeBytes"` +} + var ( attachdetachMutex = keymutex.NewKeyMutex() supportedFeatures = sets.NewString("layering") @@ -208,6 +219,21 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { return rbdVol, nil } +func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) { + var ok bool + rbdSnap := &rbdSnapshot{} + rbdSnap.Pool, ok = snapOptions["pool"] + if !ok { + return nil, fmt.Errorf("Missing required parameter pool") + } + rbdSnap.Monitors, ok = snapOptions["monitors"] + if !ok { + return nil, fmt.Errorf("Missing required parameter monitors") + } + + return rbdSnap, nil +} + func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (string, error) { var err error var output []byte @@ -376,6 +402,50 @@ func deleteVolInfo(image string, persistentStoragePath string) error { return nil } +func persistSnapInfo(snapshot string, persistentStoragePath string, snapInfo *rbdSnapshot) error { + file := path.Join(persistentStoragePath, snapshot+".json") + fp, err := os.Create(file) + if err != nil { + glog.Errorf("rbd: failed to create persistent storage file %s with error: %v\n", file, err) + return fmt.Errorf("rbd: create err %s/%s", file, err) + } + defer fp.Close() + encoder := json.NewEncoder(fp) + if err = encoder.Encode(snapInfo); err != nil { + glog.Errorf("rbd: failed to encode snapInfo: %+v for file: %s with error: %v\n", snapInfo, file, err) + return fmt.Errorf("rbd: encode err: %v", err) + } + glog.Infof("rbd: successfully saved snapInfo: %+v into file: %s\n", snapInfo, file) + return nil +} + +func loadSnapInfo(snapshot string, persistentStoragePath string, snapInfo *rbdSnapshot) error { + file := path.Join(persistentStoragePath, snapshot+".json") + fp, err := os.Open(file) + if err != nil { + return fmt.Errorf("rbd: open err %s/%s", file, err) + } + defer fp.Close() + + decoder := json.NewDecoder(fp) + if err = decoder.Decode(snapInfo); err != nil { + return fmt.Errorf("rbd: decode err: %v.", err) + } + return nil +} + +func deleteSnapInfo(snapshot string, persistentStoragePath string) error { + file := path.Join(persistentStoragePath, snapshot+".json") + glog.Infof("rbd: Deleting file for Snapshot: %s at: %s resulting path: %+v\n", snapshot, persistentStoragePath, file) + err := os.Remove(file) + if err != nil { + if err != os.ErrNotExist { + return fmt.Errorf("rbd: error removing file: %s/%s", file, err) + } + } + return nil +} + func getRBDVolumeByID(volumeID string) (rbdVolume, error) { if rbdVol, ok := rbdVolumes[volumeID]; ok { return rbdVol, nil @@ -391,3 +461,132 @@ func getRBDVolumeByName(volName string) (rbdVolume, error) { } return rbdVolume{}, fmt.Errorf("volume name %s does not exit in the volumes list", volName) } + +func getRBDSnapshotByName(snapName string) (rbdSnapshot, error) { + for _, rbdSnap := range rbdSnapshots { + if rbdSnap.SnapName == snapName { + return rbdSnap, nil + } + } + return rbdSnapshot{}, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName) +} + +func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pOpts.Monitors + image := pOpts.VolName + snapID := pOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to protect snapshot: %v, command output: %s", err, string(output)) + } + + return nil +} + +func createSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pOpts.Monitors + image := pOpts.VolName + snapID := pOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to create snapshot: %v, command output: %s", err, string(output)) + } + + return nil +} + +func unprotectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pOpts.Monitors + image := pOpts.VolName + snapID := pOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to unprotect snapshot: %v, command output: %s", err, string(output)) + } + + return nil +} + +func deleteSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pOpts.Monitors + image := pOpts.VolName + snapID := pOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to delete snapshot: %v, command output: %s", err, string(output)) + } + + return nil +} + +func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pVolOpts.Monitors + image := pVolOpts.VolName + snapID := pSnapOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, pVolOpts.Monitors, pVolOpts.Pool, RBDUserID, key) + args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to restore snapshot: %v, command output: %s", err, string(output)) + } + + return nil +}