diff --git a/powersimdata/scenario/create.py b/powersimdata/scenario/create.py index feb794449..677ed1e7f 100644 --- a/powersimdata/scenario/create.py +++ b/powersimdata/scenario/create.py @@ -93,13 +93,8 @@ def _upload_change_table(self): """Uploads change table to server.""" print("--> Writing change table on local machine") self.builder.change_table.write(self._scenario_info["id"]) - print("--> Uploading change table to server") file_name = self._scenario_info["id"] + "_ct.pkl" - self._data_access.copy_to( - file_name, server_setup.LOCAL_DIR, server_setup.INPUT_DIR - ) - print("--> Deleting change table on local machine") - os.remove(os.path.join(server_setup.LOCAL_DIR, file_name)) + self._data_access.copy_to(file_name, server_setup.INPUT_DIR) def get_ct(self): """Returns change table. diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py index c0f4e7877..65676d505 100644 --- a/powersimdata/scenario/execute.py +++ b/powersimdata/scenario/execute.py @@ -231,10 +231,13 @@ def __init__(self, data_access, scenario_info, grid, ct): self.grid = grid self.ct = ct self.server_config = server_setup.PathConfig(server_setup.DATA_ROOT_DIR) + self.scenario_folder = "scenario_%s" % scenario_info["id"] self.TMP_DIR = posixpath.join( - self.server_config.execute_dir(), - "scenario_%s" % scenario_info["id"], + self.server_config.execute_dir(), self.scenario_folder + ) + self.REL_TMP_DIR = posixpath.join( + server_setup.EXECUTE_DIR, self.scenario_folder ) def create_folder(self): @@ -366,28 +369,17 @@ def prepare_mpc_file(self): appendmat=False, ) self._data_access.copy_to( - file_name, - server_setup.LOCAL_DIR, - self.TMP_DIR, - change_name_to="case_storage.mat", + file_name, self.REL_TMP_DIR, change_name_to="case_storage.mat" ) - print("Deleting %s on local machine" % file_name) - os.remove(os.path.join(server_setup.LOCAL_DIR, file_name)) # MPC file file_name = "%s_case.mat" % self._scenario_info["id"] savemat(os.path.join(server_setup.LOCAL_DIR, file_name), mpc, appendmat=False) self._data_access.copy_to( - file_name, - server_setup.LOCAL_DIR, - self.TMP_DIR, - change_name_to="case.mat", + file_name, self.REL_TMP_DIR, change_name_to="case.mat" ) - print("Deleting %s on local machine" % file_name) - os.remove(os.path.join(server_setup.LOCAL_DIR, file_name)) - def prepare_profile(self, kind, profile_as=None): """Prepares profile for simulation. @@ -406,8 +398,7 @@ def prepare_profile(self, kind, profile_as=None): f"scenario_{profile_as}", ) to_dir = posixpath.join( - self.server_config.execute_dir(), - f"scenario_{self._scenario_info['id']}", + self.server_config.execute_dir(), self.scenario_folder ) command = f"cp {from_dir}/{kind}.csv {to_dir}" stdin, stdout, stderr = self._data_access.execute_command(command) @@ -445,18 +436,11 @@ def _prepare_scaled_profile(self, kind, profile): profile = profile.get_profile(kind) print( - "Writing scaled %s profile in %s on local machine" - % (kind, server_setup.LOCAL_DIR) + f"Writing scaled {kind} profile in {server_setup.LOCAL_DIR} on local machine" ) file_name = "%s_%s.csv" % (self._scenario_info["id"], kind) profile.to_csv(os.path.join(server_setup.LOCAL_DIR, file_name)) self._data_access.copy_to( - file_name, - server_setup.LOCAL_DIR, - self.TMP_DIR, - change_name_to="%s.csv" % kind, + file_name, self.REL_TMP_DIR, change_name_to=f"{kind}.csv" ) - - print("Deleting %s on local machine" % file_name) - os.remove(os.path.join(server_setup.LOCAL_DIR, file_name)) diff --git a/powersimdata/utility/transfer_data.py b/powersimdata/utility/transfer_data.py index 79796d8bb..646fb4839 100644 --- a/powersimdata/utility/transfer_data.py +++ b/powersimdata/utility/transfer_data.py @@ -138,46 +138,46 @@ def copy_from(self, file_name, from_dir=None): stdin, stdout, stderr = self.ssh.exec_command("ls " + from_path) if len(stderr.readlines()) != 0: raise FileNotFoundError(f"{file_name} not found in {from_dir} on server") - else: - print(f"Transferring {file_name} from server") - to_path = os.path.join(to_dir, file_name) - sftp = self.ssh.open_sftp() - try: - cbk, bar = progress_bar(ascii=True, unit="b", unit_scale=True) - sftp.get(from_path, to_path, callback=cbk) - finally: - sftp.close() - bar.close() - def copy_to(self, file_name, from_dir, to_dir, change_name_to=None): + print(f"Transferring {file_name} from server") + to_path = os.path.join(to_dir, file_name) + sftp = self.ssh.open_sftp() + try: + cbk, bar = progress_bar(ascii=True, unit="b", unit_scale=True) + sftp.get(from_path, to_path, callback=cbk) + finally: + sftp.close() + bar.close() + + def copy_to(self, file_name, to_dir, change_name_to=None): """Copy a file from userspace to data store. :param str file_name: file name to copy. - :param str from_dir: userspace directory to copy file from. :param str to_dir: data store directory to copy file to. :param str change_name_to: new name for file when copied to data store. """ - from_path = os.path.join(from_dir, file_name) + from_path = os.path.join(self.dest_root, file_name) if not os.path.isfile(from_path): raise FileNotFoundError( - f"{file_name} not found in {from_dir} on local machine" + f"{file_name} not found in {self.dest_root} on local machine" ) - else: - if bool(change_name_to): - to_path = posixpath.join(to_dir, change_name_to) - else: - to_path = posixpath.join(to_dir, file_name) - stdin, stdout, stderr = self.ssh.exec_command("ls " + to_path) - if len(stderr.readlines()) == 0: - raise IOError(f"{file_name} already exists in {to_dir} on server") - else: - print(f"Transferring {file_name} to server") - sftp = self.ssh.open_sftp() - try: - sftp.put(from_path, to_path) - finally: - sftp.close() + + file_name = file_name if change_name_to is None else change_name_to + to_path = posixpath.join(self.root, to_dir, file_name) + _, _, stderr = self.ssh.exec_command("ls " + to_path) + if len(stderr.readlines()) == 0: + raise IOError(f"{file_name} already exists in {to_dir} on server") + + print(f"Transferring {from_path} to server") + sftp = self.ssh.open_sftp() + try: + sftp.put(from_path, to_path) + finally: + sftp.close() + + print(f"--> Deleting {from_path} on local machine") + os.remove(from_path) def copy(self, src, dest, recursive=False, update=False): r_flag = "R" if recursive else ""