feat: subclass to get electrification profiles

powersimdata/data_access/data_access.py

@@ -9,11 +9,7 @@
 from fs.path import basename, dirname
 from fs.tempfs import TempFS
 
-from powersimdata.data_access.fs_helper import (
-    get_blob_fs,
-    get_multi_fs,
-    get_profile_version,
-)
+from powersimdata.data_access.fs_helper import get_blob_fs, get_multi_fs
 from powersimdata.utility import server_setup
 
 
@@ -146,16 +142,16 @@ def _check_file_exists(self, path, should_exist=True):
         if not should_exist and exists:
             raise OSError(f"{path} already exists on {location}")
 
-    def get_profile_version(self, grid_model, kind):
-        """Returns available raw profile from blob storage
+    def get_profile_version(self, callback):
+        """Returns available raw profile from blob storage or local disk
 
-        :param str grid_model: grid model.
-        :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.
+        :param callable callback: a function taking a fs instance that returns the
+            available profiles on that fs
         :return: (*list*) -- available profile version.
         """
-        bfs = fs.open_fs("azblob://besciences@profiles")
-        blob_version = get_profile_version(bfs, grid_model, kind)
-        local_version = get_profile_version(self.local_fs, grid_model, kind)
+        bfs = get_blob_fs("profiles")
+        blob_version = callback(bfs)
+        local_version = callback(self.local_fs)
         return list(set(blob_version + local_version))
 
     def checksum(self, relative_path):

powersimdata/data_access/fs_helper.py

@@ -49,22 +49,3 @@ def get_multi_fs(root):
     remotes = ",".join([f[0] for f in mfs.iterate_fs()])
     print(f"Initialized remote filesystem with {remotes}")
     return mfs
-
-
-def get_profile_version(_fs, grid_model, kind):
-    """Returns available raw profile from the given filesystem
-
-    :param fs.base.FS _fs: filesystem instance
-    :param str grid_model: grid model.
-    :param str kind: *'demand'*, *'hydro'*, *'solar'*, *'wind'*,
-        *'demand_flexibility_up'*, *'demand_flexibility_dn'*,
-        *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*.
-    :return: (*list*) -- available profile version.
-    """
-    _fs = _fs.makedirs(f"raw/{grid_model}", recreate=True)
-    matching = [f for f in _fs.listdir(".") if kind in f]
-
-    # Don't include demand flexibility profiles as possible demand profiles
-    if kind == "demand":
-        matching = [p for p in matching if "demand_flexibility" not in p]
-    return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching]

powersimdata/input/electrified_demand_input.py

@@ -0,0 +1,43 @@
+from powersimdata.input.profile_input import ProfileInput
+
+
+def get_profile_version(_fs, grid_model, kind, end_use, tech):
+    _fs = _fs.makedirs(f"raw/{grid_model}/{kind}", recreate=True)
+    base_name = f"{end_use}_{tech}_"
+    matching = [f for f in _fs.listdir(".") if base_name in f]
+
+    return [f.replace(base_name, "").replace(".csv", "") for f in matching]
+
+
+class ElectrifiedDemand(ProfileInput):
+    """Loads electrification profile data"""
+
+    def __init__(self):
+        super().__init__()
+        self._file_extension = {}
+
+    def get_profile(self, grid_model, kind, profile):
+        """Get the specified profile
+
+        :param str grid_model: the grid model
+        :param str kind: the kind of electrification
+        :param str profile: the filename
+        :return: (*pandas.DataFrame*) -- profile data frame
+        """
+        path = f"raw/{grid_model}/electrification/{kind}/{profile}.csv"
+        return self._get_data_internal(path)
+
+    def get_profile_version(self, grid_model, kind, end_use, tech):
+        """Returns available raw profile from blob storage or local disk.
+
+        :param str grid_model: grid model.
+        :param str kind: *'building'*, *'transportation'*
+        :param str end_use: electrification use case
+        :param str tech: the technology used for the given use case
+        :return: (*list*) -- available profile version.
+        """
+
+        def _callback(fs):
+            return get_profile_version(fs, grid_model, kind, end_use, tech)
+
+        return self.data_access.get_profile_version(_callback)

powersimdata/input/input_base.py

@@ -54,7 +54,9 @@ def get_data(self, scenario_info, field_name):
         print("--> Loading %s" % field_name)
 
         filepath = self._get_file_path(scenario_info, field_name)
+        return self._get_data_internal(filepath)
 
+    def _get_data_internal(self, filepath):
         key = cache_key(filepath)
         cached = _cache.get(key)
         if cached is not None:

powersimdata/input/profile_input.py

@@ -14,6 +14,25 @@
 }
 
 
+def get_profile_version(_fs, grid_model, kind):
+    """Returns available raw profile from the given filesystem
+
+    :param fs.base.FS _fs: filesystem instance
+    :param str grid_model: grid model.
+    :param str kind: *'demand'*, *'hydro'*, *'solar'*, *'wind'*,
+        *'demand_flexibility_up'*, *'demand_flexibility_dn'*,
+        *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*.
+    :return: (*list*) -- available profile version.
+    """
+    _fs = _fs.makedirs(f"raw/{grid_model}", recreate=True)
+    matching = [f for f in _fs.listdir(".") if kind in f]
+
+    # Don't include demand flexibility profiles as possible demand profiles
+    if kind == "demand":
+        matching = [p for p in matching if "demand_flexibility" not in p]
+    return [f.replace(f"{kind}_", "").replace(".csv", "") for f in matching]
+
+
 class ProfileInput(InputBase):
     """Loads profile data"""
 
@@ -59,4 +78,8 @@ def get_profile_version(self, grid_model, kind):
             *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*.
         :return: (*list*) -- available profile version.
         """
-        return self.data_access.get_profile_version(grid_model, kind)
+
+        def _callback(fs):
+            return get_profile_version(fs, grid_model, kind)
+
+        return self.data_access.get_profile_version(_callback)

powersimdata/input/tests/test_profile_input.py

@@ -1,7 +1,6 @@
 from fs.tempfs import TempFS
 
-from powersimdata.data_access.data_access import get_profile_version
-from powersimdata.input.profile_input import ProfileInput
+from powersimdata.input.profile_input import ProfileInput, get_profile_version
 
 
 def test_get_profile_version():