refactor: simplify move even more

powersimdata/data_access/data_access.py

@@ -58,7 +58,7 @@ def copy(self, src, dest):
         """Copy file to new location
 
         :param str src: path to file
-        :param str dest: destination path
+        :param str dest: destination folder
         """
         if self.fs.exists(dest) and self.fs.isdir(dest):
             dest = self.join(dest, fs2.path.basename(src))

powersimdata/scenario/move.py

@@ -1,3 +1,6 @@
+from fs.copy import copy_dir
+from fs.walk import Walker
+
 from powersimdata.data_access.data_access import get_ssh_fs
 from powersimdata.scenario.state import State
 from powersimdata.utility import server_setup
@@ -37,11 +40,8 @@ def move_scenario(self, target="disk", confirm=True):
         if target != "disk":
             raise ValueError("scenario data can only be backed up to disk now")
 
-        backup = BackUpDisk(self._data_access, self._scenario_info, confirm)
-
-        backup.move_input_data()
-        backup.move_output_data()
-        backup.move_temporary_folder()
+        backup = BackUpDisk(self._data_access, self._scenario_info)
+        backup.backup_scenario(confirm=confirm)
 
         sid = self._scenario_info["id"]
         self._execute_list_manager.set_status(sid, "moved")
@@ -55,40 +55,30 @@ class BackUpDisk:
     :param dict scenario_info: scenario information.
     """
 
-    def __init__(self, data_access, scenario_info, confirm):
+    def __init__(self, data_access, scenario_info):
         """Constructor."""
         self._data_access = data_access
         self._scenario_info = scenario_info
         self.scenario_id = self._scenario_info["id"]
-        self._fs = get_ssh_fs()
         self._join = data_access.join
-        self._confirm = confirm
-
-    def _move_data(self, pattern, folder):
-        search_path = self._join(server_setup.DATA_ROOT_DIR, folder)
-        for m in self._fs.glob(pattern, path=search_path):
-            dest = self._join(server_setup.BACKUP_DATA_ROOT_DIR, folder)
-            self._fs.makedirs(dest, recreate=True)
-            self._fs.copy(m.path, self._join(dest, m.info.name))
-            self._data_access.remove(m.path, confirm=self._confirm)
-
-    def move_input_data(self):
-        """Moves input data"""
-        print("--> Moving scenario input data to backup disk")
-        folder = self._join(*server_setup.INPUT_DIR)
-        pattern = f"**/{self.scenario_id}_*"
-        self._move_data(pattern, folder)
-
-    def move_output_data(self):
-        """Moves output data"""
-        print("--> Moving scenario output data to backup disk")
-        folder = self._join(*server_setup.OUTPUT_DIR)
-        pattern = f"**/{self.scenario_id}_*"
-        self._move_data(pattern, folder)
-
-    def move_temporary_folder(self):
-        """Moves temporary folder."""
-        print("--> Moving temporary folder to backup disk")
-        folder = self._data_access.tmp_folder(self.scenario_id)
-        pattern = "**"
-        self._move_data(pattern, folder)
+
+    def backup_scenario(self, confirm=True):
+        """Copy scenario data to backup disk and remove original
+
+        :param bool confirm: prompt before deleting each batch of files
+        """
+        _fs = get_ssh_fs()
+        src_fs, dst_fs = _fs, _fs
+        items = [
+            (self._join(*server_setup.INPUT_DIR), f"{self.scenario_id}_*"),
+            (self._join(*server_setup.OUTPUT_DIR), f"{self.scenario_id}_*"),
+            (self._data_access.tmp_folder(self.scenario_id), "**"),
+        ]
+        for folder, pattern in items:
+            print(f"--> Moving files matching {pattern} from {folder}")
+            src_path = self._join(server_setup.DATA_ROOT_DIR, folder)
+            dst_path = self._join(server_setup.BACKUP_DATA_ROOT_DIR, folder)
+            walker = Walker(filter=[pattern])
+            copy_dir(src_fs, src_path, dst_fs, dst_path, walker=walker)
+
+            self._data_access.remove(self._join(folder, pattern), confirm=confirm)