fix: makedirs if needed and minor cleanup

powersimdata/data_access/csv_store.py

@@ -77,4 +77,4 @@ def commit(self, table, checksum):
         table.to_csv(tmp_path)
         tmp_name = os.path.basename(tmp_path)
         self.data_access.push(tmp_name, checksum, rename=self._FILE_NAME)
-        # os.close(tmp_file) # not sure if we need this anymore?
+        os.close(tmp_file)

powersimdata/data_access/data_access.py

@@ -40,7 +40,6 @@ def read(self, filepath):
         :raises ValueError: if extension is unknown.
         """
         ext = os.path.basename(filepath).split(".")[-1]
-
         self._check_file_exists(filepath)
 
         with self.fs.open(filepath, mode="rb") as file_object:
@@ -63,11 +62,8 @@ def write(self, filepath, data, save_local=True):
         :param bool save_local: whether a copy should also be saved to the local filesystem, if
             such a filesystem is configured. Defaults to True.
         """
-
         self._check_file_exists(filepath, should_exist=False)
-
         print("Writing %s" % filepath)
-
         self._write(self.fs, filepath, data)
 
         if save_local and self.local_fs is not None:
@@ -81,8 +77,9 @@ def _write(self, fs, filepath, data):
         :param (*pandas.DataFrame* or *dict*) data: data to save
         :raises ValueError: if extension is unknown.
         """
-
         ext = os.path.basename(filepath).split(".")[-1]
+        dirpath = fs2.path.dirname(filepath)
+        fs.makedirs(dirpath, recreate=True)
 
         with fs.openbin(filepath, "w") as f:
             if ext == "pkl":
@@ -116,7 +113,7 @@ def copy(self, src, dest):
         :param str src: path to file
         :param str dest: destination folder
         """
-        if self.fs.exists(dest) and self.fs.isdir(dest):
+        if self.fs.isdir(dest):
             dest = self.join(dest, fs2.path.basename(src))
 
         self.fs.copy(src, dest)
@@ -148,13 +145,6 @@ def _check_file_exists(self, path, should_exist=True):
         if not should_exist and exists:
             raise OSError(f"{path} already exists on {self.description}")
 
-    def makedir(self, path):
-        """Create path in current environment
-
-        :param str path: the relative path, excluding filename
-        """
-        self.fs.makedirs(path, recreate=True)
-
     def execute_command_async(self, command):
         """Execute a command locally at the DataAccess, without waiting for completion.
 
@@ -306,7 +296,7 @@ def push(self, file_name, checksum, rename):
         backup = f"{rename}.temp"
 
         self._check_file_exists(backup, should_exist=False)
-        print(f"Transferring {backup} to server")
+        print(f"Transferring {rename} to server")
         fs2.move.move_file(self.local_fs, file_name, self.fs, backup)
 
         values = {

powersimdata/input/input_data.py

@@ -120,9 +120,6 @@ def save_change_table(self, ct, scenario_id):
         :param dict ct: a change table
         :param str scenario_id: scenario id, used for file name
         """
-
-        # note pyfilesystem path conventions:
-        # https://docs.pyfilesystem.org/en/latest/reference/path.html
         filepath = "/".join([*server_setup.INPUT_DIR, f"{scenario_id}_ct.pkl"])
         self.data_access.write(filepath, ct)
 

powersimdata/scenario/execute.py

@@ -101,7 +101,6 @@ def prepare_simulation_input(self, profiles_as=None):
             si = SimulationInput(
                 self._data_access, self._scenario_info, self.grid, self.ct
             )
-            si.create_folder()
             for p in ["demand", "hydro", "solar", "wind"]:
                 si.prepare_profile(p, profiles_as)
 
@@ -193,12 +192,6 @@ def __init__(self, data_access, scenario_info, grid, ct):
 
         self.REL_TMP_DIR = self._data_access.tmp_folder(self.scenario_id)
 
-    def create_folder(self):
-        """Creates folder on server that will enclose simulation inputs."""
-        description = self._data_access.description
-        print(f"--> Creating temporary folder on {description} for simulation inputs")
-        self._data_access.makedir(self.REL_TMP_DIR)
-
     def prepare_mpc_file(self):
         """Creates MATPOWER case file."""
         file_path = "/".join([self.REL_TMP_DIR, "case.mat"])
@@ -228,5 +221,5 @@ def prepare_profile(self, kind, profile_as=None):
             self._data_access.write(filepath, profile, save_local=False)
         else:
             from_dir = self._data_access.tmp_folder(profile_as)
-            src = self._data_access.join(from_dir, file_name)
+            src = "/".join([from_dir, file_name])
             self._data_access.copy(src, self.REL_TMP_DIR)