Feature/ub 1941 unmount fix

remote/mounter/block_device_mounter_utils/block_device_mounter_utils.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/IBM/ubiquity/remote/mounter/block_device_utils"
 	"github.com/IBM/ubiquity/resources"
+	"github.com/IBM/ubiquity/utils"
 	"github.com/IBM/ubiquity/utils/logs"
 	"github.com/nightlyone/lockfile"
 )
@@ -36,6 +37,7 @@ type blockDeviceMounterUtils struct {
 	blockDeviceUtils block_device_utils.BlockDeviceUtils
 	rescanFlock      lockfile.Lockfile
 	mpathFlock       lockfile.Lockfile
+	exec             utils.Executor
 }
 
 func NewBlockDeviceMounterUtilsWithBlockDeviceUtils(blockDeviceUtils block_device_utils.BlockDeviceUtils) BlockDeviceMounterUtils {
@@ -60,6 +62,7 @@ func newBlockDeviceMounterUtils(blockDeviceUtils block_device_utils.BlockDeviceU
 		blockDeviceUtils: blockDeviceUtils,
 		rescanFlock:      rescanLock,
 		mpathFlock:       mpathLock,
+		exec:             utils.NewExecutor(),
 	}
 }
 
@@ -142,13 +145,26 @@ func (b *blockDeviceMounterUtils) UnmountDeviceFlow(devicePath string, volumeWwn
 	defer b.mpathFlock.Unlock()
 	defer b.logger.Debug("Released mpathLock for device", logs.Args{{"device", devicePath}})
 
+	b.prepareAndStoreVolumeToCache(volumeWwn)
 	if err := b.blockDeviceUtils.Cleanup(devicePath); err != nil {
 		return b.logger.ErrorRet(err, "Cleanup failed")
 	}
 
 	return nil
 }
 
+func (b *blockDeviceMounterUtils) prepareAndStoreVolumeToCache(volumeWwn string) {
+	volumeMountProperties := &resources.VolumeMountProperties{WWN: volumeWwn}
+	if _, devMapper, devNames, err := utils.GetMultipathOutputAndDeviceMapperAndDevice(volumeWwn, b.exec); err == nil {
+		volumeMountProperties.DeviceMapper = devMapper
+		volumeMountProperties.Devices = devNames
+		// store the volumeMountProperties to a local cache, it will be used in cleanup stage.
+		b.blockDeviceUtils.StoreVolumeToCache(volumeMountProperties)
+	} else {
+		b.logger.Warning("Failed to store volume info", logs.Args{{"volumeWWN", volumeWwn}, {"error", err}})
+	}
+}
+
 // RescanAll triggers the following OS rescanning :
 // 1. iSCSI rescan (if protocol given is iscsi)
 // 2. SCSI rescan

remote/mounter/block_device_utils/block_device_utils_test.go

@@ -52,6 +52,38 @@ var _ = Describe("block_device_utils_test", func() {
 		bdUtils = block_device_utils.NewBlockDeviceUtilsWithExecutorAndConnector(fakeExec, fakeFcConnector)
 	})
 
+	Context("volume cache", func() {
+		BeforeEach(func() {
+			volumeMountProperties = &resources.VolumeMountProperties{WWN: "wwn", Devices: []string{"sda"}, DeviceMapper: "mpatha"}
+			bdUtils.StoreVolumeToCache(volumeMountProperties)
+		})
+
+		AfterEach(func() {
+			bdUtils.RemoveVolumeFromCache(volumeMountProperties)
+		})
+
+		Context("get", func() {
+
+			It("should get from cache successfully", func() {
+				volume := bdUtils.GetVolumeFromCache(&resources.VolumeMountProperties{WWN: volumeMountProperties.WWN})
+				Expect(volume.DeviceMapper).To(Equal(volumeMountProperties.DeviceMapper))
+				Expect(volume.Devices).To(Equal(volumeMountProperties.Devices))
+			})
+		})
+
+		Context("remove", func() {
+
+			It("should remove from cache successfully", func() {
+				volume := bdUtils.GetVolumeFromCache(&resources.VolumeMountProperties{WWN: volumeMountProperties.WWN})
+				Expect(volume.DeviceMapper).To(Equal(volumeMountProperties.DeviceMapper))
+				Expect(volume.Devices).To(Equal(volumeMountProperties.Devices))
+				bdUtils.RemoveVolumeFromCache(volume)
+				volumeAgain := bdUtils.GetVolumeFromCache(&resources.VolumeMountProperties{WWN: volumeMountProperties.WWN})
+				Expect(volumeAgain).To(BeNil())
+			})
+		})
+	})
+
 	Context(".Rescan", func() {
 		It("Rescan ISCSI calls 'sudo iscsiadm -m session --rescan'", func() {
 			err = bdUtils.Rescan(block_device_utils.ISCSI, volumeMountProperties)
@@ -91,6 +123,10 @@ var _ = Describe("block_device_utils_test", func() {
 		})
 	})
 	Context(".CleanupDevices", func() {
+		BeforeEach(func() {
+			bdUtils.StoreVolumeToCache(volumeMountProperties)
+		})
+
 		It("Cleanup ISCSI calls 'sudo iscsiadm -m session --rescan'", func() {
 			err = bdUtils.CleanupDevices(block_device_utils.ISCSI, volumeMountProperties)
 			Expect(err).ToNot(HaveOccurred())

remote/mounter/block_device_utils/rescan.go

@@ -18,6 +18,7 @@ package block_device_utils
 
 import (
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/IBM/ubiquity/resources"
@@ -28,9 +29,38 @@ import (
 const rescanIscsiTimeout = 1 * 60 * 1000
 const rescanScsiTimeout = 2 * 60 * 1000
 
+// store the volume mount info to a local cache.
+// It is just a workaround, We can not get the multipath devices from multipath -ll
+// output in the cleanup stage. because we run multipath -f before it.
+var volumeCache = &sync.Map{}
+
 var FcHostDir = "/sys/class/fc_host/"
 var ScsiHostDir = "/sys/class/scsi_host/"
 
+func (b *blockDeviceUtils) GetVolumeFromCache(volumeMountProperties *resources.VolumeMountProperties) *resources.VolumeMountProperties {
+	defer b.logger.Trace(logs.DEBUG, logs.Args{{"wwn", volumeMountProperties.WWN}})()
+
+	if volume, exists := volumeCache.Load(volumeMountProperties.WWN); exists {
+		return volume.(*resources.VolumeMountProperties)
+	}
+	b.logger.Warning("Volume not found in cache.", logs.Args{{"wwn", volumeMountProperties.WWN}})
+	return nil
+}
+
+func (b *blockDeviceUtils) RemoveVolumeFromCache(volumeMountProperties *resources.VolumeMountProperties) {
+	defer b.logger.Trace(logs.DEBUG, logs.Args{{"wwn", volumeMountProperties.WWN}})()
+
+	volumeCache.Delete(volumeMountProperties.WWN)
+}
+
+func (b *blockDeviceUtils) StoreVolumeToCache(volumeMountProperties *resources.VolumeMountProperties) {
+	defer b.logger.Trace(logs.DEBUG, logs.Args{{"wwn", volumeMountProperties.WWN}})()
+
+	volume := new(resources.VolumeMountProperties)
+	*volume = *volumeMountProperties
+	volumeCache.Store(volumeMountProperties.WWN, volume)
+}
+
 func (b *blockDeviceUtils) Rescan(protocol Protocol, volumeMountProperties *resources.VolumeMountProperties) error {
 	defer b.logger.Trace(logs.DEBUG)()
 
@@ -103,5 +133,17 @@ func (b *blockDeviceUtils) CleanupISCSIDevices() error {
 }
 
 func (b *blockDeviceUtils) CleanupSCSIDevices(volumeMountProperties *resources.VolumeMountProperties) error {
-	return b.fcConnector.DisconnectVolume(volumeMountProperties)
+	defer b.logger.Trace(logs.DEBUG)()
+
+	volume := b.GetVolumeFromCache(volumeMountProperties)
+	if volume == nil {
+		// devices are already cleaned up
+		return nil
+	}
+
+	if err := b.fcConnector.DisconnectVolume(volume); err != nil {
+		return err
+	}
+	b.RemoveVolumeFromCache(volumeMountProperties)
+	return nil
 }

remote/mounter/block_device_utils/resources.go

@@ -43,4 +43,7 @@ type BlockDeviceUtils interface {
 	IsDeviceMounted(devPath string) (bool, []string, error)
 	IsDirAMountPoint(dirPath string) (bool, []string, error)
 	SetDmsetup(mpath string) error
+	GetVolumeFromCache(volumeMountProperties *resources.VolumeMountProperties) *resources.VolumeMountProperties
+	RemoveVolumeFromCache(volumeMountProperties *resources.VolumeMountProperties)
+	StoreVolumeToCache(volumeMountProperties *resources.VolumeMountProperties)
 }