[#323] Unity: add advanced dedup support (#324)

Add Advanced Deduplication support for Unity LUNs.

storops/exception.py

@@ -1485,3 +1485,23 @@ class UnityQoSMaxKBPSOutOfRangeError(UnityReplicationError):
 
 class UnitySnapScheduleNameInUseError(UnityException):
     pass
+
+
+@rest_exception
+class UnityAdvancedDedupNotSupportedError(UnityException):
+    error_code = 108009040
+
+
+@rest_exception
+class UnityAdvancedDedupRequireCompressionEnabledError(UnityException):
+    error_code = 108007752
+
+
+@rest_exception
+class UnityCompressionRequireLunIsThinError(UnityException):
+    error_code = 108007750
+
+
+@rest_exception
+class UnityCompressionRequireAllFlashPoolError(UnityException):
+    error_code = 108009014

storops/lib/models.py

@@ -0,0 +1,54 @@
+# coding=utf-8
+# Copyright (c) 2017 Dell Inc. or its subsidiaries.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import operator
+
+
+class UnityModel:
+    def __init__(self, model):
+        self.model = model.split(' ')[-1]
+        self.series = self.model[0]
+
+        if self.model.endswith('F'):
+            self.is_all_flash = True
+            self.model_base = self.model.rstrip('F')
+        else:
+            self.is_all_flash = False
+            self.model_base = self.model
+
+    def is_same_series(self, other):
+        return (self.series == other.series) and (
+                self.is_all_flash == other.is_all_flash)
+
+    def compare(self, other, op):
+        if not self.is_same_series(other):
+            return False
+        return op(int(self.model_base), int(other.model_base))
+
+    def __eq__(self, other):
+        return self.compare(other, operator.eq)
+
+    def __gt__(self, other):
+        return self.compare(other, operator.gt)
+
+    def __ge__(self, other):
+        return self.compare(other, operator.ge)
+
+    def __lt__(self, other):
+        return self.compare(other, operator.lt)
+
+    def __le__(self, other):
+        return self.compare(other, operator.le)

storops/unity/enums.py

@@ -1207,6 +1207,12 @@ def get_enum_class(cls):
         return DayOfWeekEnum
 
 
+class DataReductionStatusEnum(UnityEnum):
+    DISABLED = (0, 'Disabled')
+    ENABLED = (1, 'Enabled')
+    MIXED = (65535, 'Mixed')
+
+
 class FSRenamePolicyEnum(UnityEnum):
     ALL_RENAME_ALLOWED = (0, 'All_Rename_Allowed')
     SMB_RENAME_FORBIDDEN = (1, 'SMB_Rename_Forbidden')

storops/unity/parser_configs.yaml

@@ -42,6 +42,7 @@ UnityLun:
     - label: isThinEnabled
     - label: isCompressionEnabled
     - label: isDataReductionEnabled
+    - label: isAdvancedDedupEnabled
     - label: storageResource
       converter: UnityStorageResource
     - label: pool
@@ -73,6 +74,9 @@ UnityLun:
       converter: UnityLun
     - label: familyCloneCount
     - label: isThinClone
+    - label: dataReductionSizeSaved
+    - label: dataReductionPercent
+    - label: dataReductionRatio
 
 
 UnityHealth:
@@ -150,6 +154,13 @@ UnityStorageResource: &UnityStorageResource
       converter: UnityHostVvolDatastoreList
     - label: virtualVolumes
       converter: UnityVirtualVolumeList
+    - label: dataReductionStatus
+      converter: DataReductionStatusEnum
+    - label: advancedDedupStatus
+      converter: DedupStatusEnum
+    - label: dataReductionSizeSaved
+    - label: dataReductionPercent
+    - label: dataReductionRatio
 
 
 UnityConsistencyGroup:

storops/unity/resource/lun.py

@@ -76,7 +76,8 @@ def make_compression_body(cli=None,
         fastVPParameters=UnityClient.make_body(
             tieringPolicy=kwargs.get('tiering_policy')),
         ioLimitParameters=UnityClient.make_body(
-            ioLimitPolicy=kwargs.get('io_limit_policy')))
+            ioLimitPolicy=kwargs.get('io_limit_policy')),
+        isAdvancedDedupEnabled=kwargs.get('is_advanced_dedup_enabled'))
 
     compression_body = make_compression_body(
         cli,
@@ -114,7 +115,7 @@ def create(cls, cli, name, pool, size, sp=None, host_access=None,
                is_repl_dst=None, tiering_policy=None, snap_schedule=None,
                is_snap_schedule_paused=None, skip_sync_to_remote_system=None,
                is_compression=None, create_vmfs=False, major_version=None,
-               block_size=None):
+               block_size=None, is_advanced_dedup_enabled=None):
         pool_clz = storops.unity.resource.pool.UnityPool
         pool = pool_clz.get(cli, pool)
 
@@ -126,7 +127,8 @@ def create(cls, cli, name, pool, size, sp=None, host_access=None,
             is_snap_schedule_paused=is_snap_schedule_paused,
             skip_sync_to_remote_system=skip_sync_to_remote_system,
             is_compression=is_compression, major_version=major_version,
-            block_size=block_size)
+            block_size=block_size,
+            is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
         create_method = 'createVmwareLun' if create_vmfs else 'createLun'
 
@@ -250,7 +252,8 @@ def modify(self, name=None, size=None, host_access=None,
                description=None, sp=None, io_limit_policy=None,
                is_repl_dst=None, tiering_policy=None, snap_schedule=None,
                is_snap_schedule_paused=None, skip_sync_to_remote_system=None,
-               is_compression=None, major_version=None, block_size=None):
+               is_compression=None, major_version=None, block_size=None,
+               is_advanced_dedup_enabled=None):
         if self.is_cg_member:
             if any(each is not None for each in [is_repl_dst, snap_schedule,
                                                  is_snap_schedule_paused,
@@ -274,7 +277,8 @@ def modify(self, name=None, size=None, host_access=None,
                 is_snap_schedule_paused=is_snap_schedule_paused,
                 skip_sync_to_remote_system=skip_sync_to_remote_system,
                 is_compression=is_compression, major_version=major_version,
-                block_size=block_size)
+                block_size=block_size,
+                is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
             if self.is_vmware_vmfs:
                 resp = self._cli.action(UnityStorageResource().resource_class,
@@ -407,24 +411,43 @@ def thin_clone(self, name, io_limit_policy=None, description=None):
         return TCHelper.thin_clone(self._cli, self, name, io_limit_policy,
                                    description)
 
-    def _is_move_session_supported(self, dest):
+    def _is_move_session_supported(self, dest, is_compressed=None,
+                                   is_advanced_dedup_enabled=None):
         if self.is_thin_clone:
             log.error('Not support move session, source lun is thin clone.')
             return False
-        if self.is_data_reduction_enabled and not dest.is_all_flash:
-            log.error('Not support move session, source lun is compressed, '
-                      'but destination pool is not all flash pool.')
+        if is_compressed and not dest.is_compression_supported():
+            log.error('Not support move session, target lun is compressed, '
+                      'but destination pool is not supported compression.')
+            return False
+        if (is_advanced_dedup_enabled and
+                not dest.is_advanced_dedup_supported()):
+            log.error('Not support move session, target lun is advanced '
+                      'deduplication enabled, but destination pool is not '
+                      'supported advanced deduplication.')
             return False
         return True
 
     def migrate(self, dest, **kwargs):
-        if not self._is_move_session_supported(dest):
-            return False
-
         interval = kwargs.pop('interval', 5)
         timeout = kwargs.pop('timeout', 1800)
-        is_thin = kwargs.get('is_thin')
-        is_compressed = kwargs.get('is_compressed')
+        is_thin = kwargs.get('is_thin', self.is_thin_enabled)
+
+        if is_thin:
+            is_compressed = kwargs.get('is_compressed',
+                                       self.is_data_reduction_enabled)
+        else:
+            is_compressed = False
+
+        if is_thin and is_compressed:
+            is_advanced_dedup_enabled = kwargs.get(
+                'is_advanced_dedup_enabled', self.is_advanced_dedup_enabled)
+        else:
+            is_advanced_dedup_enabled = False
+
+        if not self._is_move_session_supported(dest, is_compressed,
+                                               is_advanced_dedup_enabled):
+            return False
 
         @retryz.retry(timeout=timeout, wait=interval,
                       on_return=lambda x: not isinstance(x, bool))
@@ -437,12 +460,12 @@ def _do_check_move_session(move_session_id):
                 return False
 
         clz = storops.unity.resource.move_session.UnityMoveSession
-        if is_compressed not in (True, False):
-            is_compressed = self.is_data_reduction_enabled
         try:
-            move_session = clz.create(self._cli, self, dest,
-                                      is_data_reduction_applied=is_compressed,
-                                      is_dest_thin=is_thin)
+            move_session = clz.create(
+                self._cli, self, dest,
+                is_data_reduction_applied=is_compressed,
+                is_dest_thin=is_thin,
+                is_advanced_dedup_applied=is_advanced_dedup_enabled)
             return _do_check_move_session(move_session.id)
         except UnityMigrationSourceHasThinCloneError:
             log.error('Not support move session, source lun has thin clone.')

storops/unity/resource/move_session.py

@@ -28,13 +28,15 @@ class UnityMoveSession(UnityResource):
     @classmethod
     def create(cls, cli, source_storage_resource, destination_pool,
                source_member_lun=None, is_dest_thin=None,
-               is_data_reduction_applied=None, priority=None):
+               is_data_reduction_applied=None, is_advanced_dedup_applied=None,
+               priority=None):
         req_body = cls._compose_move_session_parameter(
             cli, source_storage_resource=source_storage_resource,
             destination_pool=destination_pool,
             source_member_lun=source_member_lun,
             is_dest_thin=is_dest_thin,
             is_data_reduction_applied=is_data_reduction_applied,
+            is_advanced_dedup_applied=is_advanced_dedup_applied,
             priority=priority)
         resp = cli.post(cls().resource_class, **req_body)
         resp.raise_if_err()
@@ -59,13 +61,15 @@ def _compose_move_session_parameter(cli, source_storage_resource=None,
                                         source_member_lun=None,
                                         is_dest_thin=None,
                                         is_data_reduction_applied=None,
+                                        is_advanced_dedup_applied=None,
                                         priority=None):
         req_body = cli.make_body(
             sourceStorageResource=source_storage_resource,
             destinationPool=destination_pool,
             sourceMemberLun=source_member_lun,
             isDestThin=is_dest_thin,
             isDataReductionApplied=is_data_reduction_applied,
+            isAdvancedDedupApplied=is_advanced_dedup_applied,
             priority=priority)
         return req_body
 

storops/unity/resource/pool.py

@@ -21,6 +21,8 @@
 import bitmath
 
 import storops.unity.resource.filesystem
+from storops.lib.models import UnityModel
+from storops.lib.version import version
 from storops.unity.resource import UnityResource, \
     UnityAttributeResource, UnityResourceList
 from storops.unity.resource.disk import UnityDiskGroup, UnityDiskList
@@ -109,7 +111,8 @@ def create_lun(self, lun_name=None, size_gb=1, sp=None, host_access=None,
                    is_repl_dst=None, snap_schedule=None,
                    is_snap_schedule_paused=None,
                    skip_sync_to_remote_system=None,
-                   io_limit_policy=None, is_compression=None):
+                   io_limit_policy=None, is_compression=None,
+                   is_advanced_dedup_enabled=None):
         size = int(bitmath.GiB(size_gb).to_Byte().value)
         return UnityLun.create(
             self._cli, lun_name, self, size, sp=sp,
@@ -121,7 +124,8 @@ def create_lun(self, lun_name=None, size_gb=1, sp=None, host_access=None,
             is_snap_schedule_paused=is_snap_schedule_paused,
             skip_sync_to_remote_system=skip_sync_to_remote_system,
             io_limit_policy=io_limit_policy,
-            is_compression=is_compression)
+            is_compression=is_compression,
+            is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
     def create_vmfs(self, vmfs_name=None, size_gb=1, sp=None, host_access=None,
                     is_thin=None, description=None, tiering_policy=None,
@@ -234,6 +238,34 @@ def disk_groups(self):
                 dgs[pd.disk_group.get_id()] = [pd]
         return dgs
 
+    def is_compression_supported(self):
+        return self.is_all_flash
+
+    def _is_advanced_dedup_supported(self, support_matrix):
+        from storops.unity.resource.system import UnitySystem
+        unity_system = UnitySystem(cli=self._cli)
+        supported = False
+        for supported_model in support_matrix:
+            if UnityModel(unity_system.model) >= UnityModel(supported_model):
+                supported = True
+                break
+        return supported and self.is_all_flash
+
+    @version('<4.5')
+    def is_advanced_dedup_supported(self):
+        return False
+
+    @version('>=4.5')  # noqa
+    def is_advanced_dedup_supported(self):
+        support_matrix = ['450F', '550F', '650F']
+        return self._is_advanced_dedup_supported(support_matrix)
+
+    @version('>=5.0')  # noqa
+    def is_advanced_dedup_supported(self):
+        support_matrix = ['380', '480', '680', '880',
+                          '380F', '450F', '550F', '650F', '880F']
+        return self._is_advanced_dedup_supported(support_matrix)
+
 
 class UnityPoolList(UnityResourceList):
     @classmethod