refactor: use relative path in copy_to and handle cleanup

powersimdata/scenario/create.py

@@ -93,13 +93,8 @@ def _upload_change_table(self):
         """Uploads change table to server."""
         print("--> Writing change table on local machine")
         self.builder.change_table.write(self._scenario_info["id"])
-        print("--> Uploading change table to server")
         file_name = self._scenario_info["id"] + "_ct.pkl"
-        self._data_access.copy_to(
-            file_name, server_setup.LOCAL_DIR, server_setup.INPUT_DIR
-        )
-        print("--> Deleting change table on local machine")
-        os.remove(os.path.join(server_setup.LOCAL_DIR, file_name))
+        self._data_access.copy_to(file_name, server_setup.INPUT_DIR)
 
     def get_ct(self):
         """Returns change table.

powersimdata/scenario/execute.py

@@ -231,10 +231,13 @@ def __init__(self, data_access, scenario_info, grid, ct):
         self.grid = grid
         self.ct = ct
         self.server_config = server_setup.PathConfig(server_setup.DATA_ROOT_DIR)
+        self.scenario_folder = "scenario_%s" % scenario_info["id"]
 
         self.TMP_DIR = posixpath.join(
-            self.server_config.execute_dir(),
-            "scenario_%s" % scenario_info["id"],
+            self.server_config.execute_dir(), self.scenario_folder
+        )
+        self.REL_TMP_DIR = posixpath.join(
+            server_setup.EXECUTE_DIR, self.scenario_folder
         )
 
     def create_folder(self):
@@ -366,28 +369,17 @@ def prepare_mpc_file(self):
                 appendmat=False,
             )
             self._data_access.copy_to(
-                file_name,
-                server_setup.LOCAL_DIR,
-                self.TMP_DIR,
-                change_name_to="case_storage.mat",
+                file_name, self.REL_TMP_DIR, change_name_to="case_storage.mat"
             )
-            print("Deleting %s on local machine" % file_name)
-            os.remove(os.path.join(server_setup.LOCAL_DIR, file_name))
 
         # MPC file
         file_name = "%s_case.mat" % self._scenario_info["id"]
         savemat(os.path.join(server_setup.LOCAL_DIR, file_name), mpc, appendmat=False)
 
         self._data_access.copy_to(
-            file_name,
-            server_setup.LOCAL_DIR,
-            self.TMP_DIR,
-            change_name_to="case.mat",
+            file_name, self.REL_TMP_DIR, change_name_to="case.mat"
         )
 
-        print("Deleting %s on local machine" % file_name)
-        os.remove(os.path.join(server_setup.LOCAL_DIR, file_name))
-
     def prepare_profile(self, kind, profile_as=None):
         """Prepares profile for simulation.
 
@@ -406,8 +398,7 @@ def prepare_profile(self, kind, profile_as=None):
                 f"scenario_{profile_as}",
             )
             to_dir = posixpath.join(
-                self.server_config.execute_dir(),
-                f"scenario_{self._scenario_info['id']}",
+                self.server_config.execute_dir(), self.scenario_folder
             )
             command = f"cp {from_dir}/{kind}.csv {to_dir}"
             stdin, stdout, stderr = self._data_access.execute_command(command)
@@ -445,18 +436,11 @@ def _prepare_scaled_profile(self, kind, profile):
         profile = profile.get_profile(kind)
 
         print(
-            "Writing scaled %s profile in %s on local machine"
-            % (kind, server_setup.LOCAL_DIR)
+            f"Writing scaled {kind} profile in {server_setup.LOCAL_DIR} on local machine"
         )
         file_name = "%s_%s.csv" % (self._scenario_info["id"], kind)
         profile.to_csv(os.path.join(server_setup.LOCAL_DIR, file_name))
 
         self._data_access.copy_to(
-            file_name,
-            server_setup.LOCAL_DIR,
-            self.TMP_DIR,
-            change_name_to="%s.csv" % kind,
+            file_name, self.REL_TMP_DIR, change_name_to=f"{kind}.csv"
         )
-
-        print("Deleting %s on local machine" % file_name)
-        os.remove(os.path.join(server_setup.LOCAL_DIR, file_name))

powersimdata/utility/transfer_data.py

@@ -138,46 +138,46 @@ def copy_from(self, file_name, from_dir=None):
         stdin, stdout, stderr = self.ssh.exec_command("ls " + from_path)
         if len(stderr.readlines()) != 0:
             raise FileNotFoundError(f"{file_name} not found in {from_dir} on server")
-        else:
-            print(f"Transferring {file_name} from server")
-            to_path = os.path.join(to_dir, file_name)
-            sftp = self.ssh.open_sftp()
-            try:
-                cbk, bar = progress_bar(ascii=True, unit="b", unit_scale=True)
-                sftp.get(from_path, to_path, callback=cbk)
-            finally:
-                sftp.close()
-                bar.close()
 
-    def copy_to(self, file_name, from_dir, to_dir, change_name_to=None):
+        print(f"Transferring {file_name} from server")
+        to_path = os.path.join(to_dir, file_name)
+        sftp = self.ssh.open_sftp()
+        try:
+            cbk, bar = progress_bar(ascii=True, unit="b", unit_scale=True)
+            sftp.get(from_path, to_path, callback=cbk)
+        finally:
+            sftp.close()
+            bar.close()
+
+    def copy_to(self, file_name, to_dir, change_name_to=None):
         """Copy a file from userspace to data store.
 
         :param str file_name: file name to copy.
-        :param str from_dir: userspace directory to copy file from.
         :param str to_dir: data store directory to copy file to.
         :param str change_name_to: new name for file when copied to data store.
         """
-        from_path = os.path.join(from_dir, file_name)
+        from_path = os.path.join(self.dest_root, file_name)
 
         if not os.path.isfile(from_path):
             raise FileNotFoundError(
-                f"{file_name} not found in {from_dir} on local machine"
+                f"{file_name} not found in {self.dest_root} on local machine"
             )
-        else:
-            if bool(change_name_to):
-                to_path = posixpath.join(to_dir, change_name_to)
-            else:
-                to_path = posixpath.join(to_dir, file_name)
-            stdin, stdout, stderr = self.ssh.exec_command("ls " + to_path)
-            if len(stderr.readlines()) == 0:
-                raise IOError(f"{file_name} already exists in {to_dir} on server")
-            else:
-                print(f"Transferring {file_name} to server")
-                sftp = self.ssh.open_sftp()
-                try:
-                    sftp.put(from_path, to_path)
-                finally:
-                    sftp.close()
+
+        file_name = file_name if change_name_to is None else change_name_to
+        to_path = posixpath.join(self.root, to_dir, file_name)
+        _, _, stderr = self.ssh.exec_command("ls " + to_path)
+        if len(stderr.readlines()) == 0:
+            raise IOError(f"{file_name} already exists in {to_dir} on server")
+
+        print(f"Transferring {from_path} to server")
+        sftp = self.ssh.open_sftp()
+        try:
+            sftp.put(from_path, to_path)
+        finally:
+            sftp.close()
+
+        print(f"--> Deleting {from_path} on local machine")
+        os.remove(from_path)
 
     def copy(self, src, dest, recursive=False, update=False):
         r_flag = "R" if recursive else ""