diff --git a/powersimdata/input/input_data.py b/powersimdata/input/input_data.py index ae7c89480..2cbba9b2c 100644 --- a/powersimdata/input/input_data.py +++ b/powersimdata/input/input_data.py @@ -68,8 +68,9 @@ def get_data(self, scenario_info, field_name): file_name = scenario_info["id"] + "_" + field_name + "." + ext from_dir = server_setup.INPUT_DIR + filepath = os.path.join(server_setup.LOCAL_DIR, from_dir, file_name) try: - return _read_data(file_name, path_to_file=from_dir) + return _read_data(filepath) except FileNotFoundError: print( "%s not found in %s on local machine" @@ -77,20 +78,19 @@ def get_data(self, scenario_info, field_name): ) self.data_access.copy_from(file_name, from_dir) - return _read_data(file_name, path_to_file=from_dir) + return _read_data(filepath) -def _read_data(file_name, path_to_file): - """Reads data. +def _read_data(filepath): + """Reads data from local machine. - :param str file_name: file name, extension either 'pkl', 'csv', or 'mat'. + :param str filepath: path to file, with extension either 'pkl', 'csv', or 'mat'. :return: (*pandas.DataFrame*, *dict*, or *str*) -- demand, hydro, solar or wind as a data frame, change table as a dict, or str containing a local path to a matfile of grid data. :raises ValueError: if extension is unknown. """ - ext = file_name.split(".")[-1] - filepath = os.path.join(server_setup.LOCAL_DIR, path_to_file, file_name) + ext = os.path.basename(filepath).split(".")[-1] if ext == "pkl": data = pd.read_pickle(filepath) elif ext == "csv": diff --git a/powersimdata/output/output_data.py b/powersimdata/output/output_data.py index 74733b7b1..da7c345f1 100644 --- a/powersimdata/output/output_data.py +++ b/powersimdata/output/output_data.py @@ -40,30 +40,18 @@ def get_data(self, scenario_id, field_name): print("--> Loading %s" % field_name) file_name = scenario_id + "_" + field_name + ".pkl" from_dir = server_setup.OUTPUT_DIR + filepath = os.path.join(server_setup.LOCAL_DIR, from_dir, file_name) try: - return _read_data(file_name, path_to_file=from_dir) + return pd.read_pickle(filepath) except pickle.UnpicklingError: err_msg = f"Unable to unpickle {file_name}, possibly corrupted in download." raise ValueError(err_msg) except FileNotFoundError: - print( - "%s not found in %s on local machine" - % (file_name, os.path.join(server_setup.LOCAL_DIR, from_dir)) - ) + print(f"{filepath} not found on local machine") self._data_access.copy_from(file_name, from_dir) - return _read_data(file_name, path_to_file=from_dir) - - -def _read_data(file_name, path_to_file): - """Reads data. - - :param str file_name: file name - :param str path_to_file: relative path segment excluding root and filename - :return: (*pandas.DataFrame*) -- specified file as a data frame. - """ - return pd.read_pickle(os.path.join(server_setup.LOCAL_DIR, path_to_file, file_name)) + return pd.read_pickle(filepath) def _check_field(field_name): diff --git a/powersimdata/scenario/delete.py b/powersimdata/scenario/delete.py index e9a9d46b0..3204867d9 100644 --- a/powersimdata/scenario/delete.py +++ b/powersimdata/scenario/delete.py @@ -1,5 +1,6 @@ import glob import os +import posixpath from powersimdata.scenario.state import State from powersimdata.utility import server_setup @@ -38,9 +39,8 @@ def delete_scenario(self): # Delete links to base profiles on server print("--> Deleting scenario input data on server") - target = "%s/%s_*" % ( - self.path_config.input_dir(), - self._scenario_info["id"], + target = posixpath.join( + self.path_config.input_dir(), "%s_*" % (self._scenario_info["id"]) ) _, _, stderr = self._data_access.remove(target, recursive=False, force=True) if len(stderr.readlines()) != 0: @@ -48,9 +48,8 @@ def delete_scenario(self): # Delete output profiles print("--> Deleting scenario output data on server") - target = "%s/%s_*" % ( - self.path_config.output_dir(), - self._scenario_info["id"], + target = posixpath.join( + self.path_config.output_dir(), "%s_*" % (self._scenario_info["id"]) ) _, _, stderr = self._data_access.remove(target, recursive=False, force=True) if len(stderr.readlines()) != 0: @@ -58,9 +57,8 @@ def delete_scenario(self): # Delete temporary folder enclosing simulation inputs print("--> Deleting temporary folder on server") - tmp_dir = "%s/scenario_%s" % ( - self.path_config.execute_dir(), - self._scenario_info["id"], + tmp_dir = posixpath.join( + self.path_config.execute_dir(), "scenario_%s" % (self._scenario_info["id"]) ) _, _, stderr = self._data_access.remove(tmp_dir, recursive=True, force=True) if len(stderr.readlines()) != 0: