Merge pull request #643 from Breakthrough-Energy/develop

develop -> master

Pipfile

@@ -18,6 +18,6 @@ paramiko = "==2.10.1"
 scipy = "~=1.5"
 tqdm = "==4.29.1"
 requests = "~=2.25"
-fs = "*"
+fs = "==2.4.14"
 "fs.sshfs" = "*"
 fs-azureblob = "*"

powersimdata/data_access/csv_store.py

@@ -44,21 +44,27 @@ def __init__(self, data_access):
         self.data_access = data_access
 
     def get_table(self):
-        """Read the given file from the server, falling back to local copy if
-        unable to connect.
+        """Attempt to download the file from server and blob storage, falling back to
+        local copy if one exists, and return the combined result.
 
         :return: (*pandas.DataFrame*) -- the specified table as a data frame.
         """
         filename = self._FILE_NAME
-        try:
-            return self._get_table(filename)
-        except:  # noqa
-            return self._get_table(filename + ".2")
+        orig = self._get_table(filename)
+        blob = self._get_table(filename + ".2")
+        df = pd.concat([orig, blob])
+        return df[~df.index.duplicated()]
 
     def _get_table(self, filename):
-        self.data_access.copy_from(filename)
-        with self.data_access.get(filename) as (f, _):
-            return _parse_csv(f)
+        try:
+            self.data_access.copy_from(filename)
+        except:  # noqa
+            pass
+        try:
+            with self.data_access.get(filename) as (f, _):
+                return _parse_csv(f)
+        except:  # noqa
+            return pd.DataFrame()
 
     def commit(self, table, checksum):
         """Save to local directory and upload if needed

powersimdata/data_access/data_access.py

@@ -9,11 +9,7 @@
 from fs.path import basename, dirname
 from fs.tempfs import TempFS
 
-from powersimdata.data_access.fs_helper import (
-    get_blob_fs,
-    get_multi_fs,
-    get_profile_version,
-)
+from powersimdata.data_access.fs_helper import get_blob_fs, get_multi_fs
 from powersimdata.utility import server_setup
 
 
@@ -97,17 +93,6 @@ def tmp_folder(self, scenario_id):
         """
         return self.join(server_setup.EXECUTE_DIR, f"scenario_{scenario_id}")
 
-    def copy(self, src, dest):
-        """Copy file to new location
-
-        :param str src: path to file
-        :param str dest: destination folder
-        """
-        if self.fs.isdir(dest):
-            dest = self.join(dest, fs.path.basename(src))
-
-        self.fs.copy(src, dest)
-
     def remove(self, base_dir, pattern, confirm=True):
         """Delete files in current environment
 
@@ -146,16 +131,16 @@ def _check_file_exists(self, path, should_exist=True):
         if not should_exist and exists:
             raise OSError(f"{path} already exists on {location}")
 
-    def get_profile_version(self, grid_model, kind):
-        """Returns available raw profile from blob storage
+    def get_profile_version(self, callback):
+        """Returns available raw profile from blob storage or local disk
 
-        :param str grid_model: grid model.
-        :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.
+        :param callable callback: a function taking a fs instance that returns the
+            available profiles on that fs
         :return: (*list*) -- available profile version.
         """
-        bfs = fs.open_fs("azblob://besciences@profiles")
-        blob_version = get_profile_version(bfs, grid_model, kind)
-        local_version = get_profile_version(self.local_fs, grid_model, kind)
+        bfs = get_blob_fs("profiles")
+        blob_version = callback(bfs)
+        local_version = callback(self.local_fs)
         return list(set(blob_version + local_version))
 
     def checksum(self, relative_path):

powersimdata/data_access/fs_helper.py

@@ -49,22 +49,3 @@ def get_multi_fs(root):
     remotes = ",".join([f[0] for f in mfs.iterate_fs()])
     print(f"Initialized remote filesystem with {remotes}")
     return mfs
-
-
-def get_profile_version(_fs, grid_model, kind):
-    """Returns available raw profile from the given filesystem
-
-    :param fs.base.FS _fs: filesystem instance
-    :param str grid_model: grid model.
-    :param str kind: *'demand'*, *'hydro'*, *'solar'*, *'wind'*,
-        *'demand_flexibility_up'*, *'demand_flexibility_dn'*,
-        *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*.
-    :return: (*list*) -- available profile version.
-    """
-    _fs = _fs.makedirs(f"raw/{grid_model}", recreate=True)
-    matching = [f for f in _fs.listdir(".") if kind in f]
-
-    # Don't include demand flexibility profiles as possible demand profiles
-    if kind == "demand":
-        matching = [p for p in matching if "demand_flexibility" not in p]
-    return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching]

powersimdata/design/generation/clean_capacity_scaling.py

@@ -2,7 +2,6 @@
 import pandas as pd
 
 from powersimdata.design.mimic_grid import mimic_generation_capacity
-from powersimdata.network.model import area_to_loadzone
 from powersimdata.scenario.scenario import Scenario
 
 
@@ -44,8 +43,8 @@ def _apply_zone_scale_factor_to_ct(ct, fuel, zone_id, scale_factor):
 
 
 def load_targets_from_csv(filename, drop_ignored=True):
-    """Interprets a CSV file as a set of targets, ensuring that required columns are present,
-    and filling in default values for optional columns.
+    """Interprets a CSV file as a set of targets, ensuring that required columns are
+    present, and filling in default values for optional columns.
 
     :param str filename: filepath to targets csv.
     :param bool drop_ignored: if True, drop all ignored columns from output.
@@ -100,12 +99,11 @@ def _make_zonename2target(grid, targets):
     :raises ValueError: if a zone is not present in any target areas, or
         if a zone is present in more than one target area.
     """
-    grid_model = grid.grid_model
     target_zones = {
-        target_name: area_to_loadzone(grid_model, target_name)
+        target_name: grid.model_immutables.area_to_loadzone(target_name)
         if pd.isnull(targets.loc[target_name, "area_type"])
-        else area_to_loadzone(
-            grid_model, target_name, targets.loc[target_name, "area_type"]
+        else grid.model_immutables.area_to_loadzone(
+            target_name, area_type=targets.loc[target_name, "area_type"]
         )
         for target_name in targets.index.tolist()
     }
@@ -379,8 +377,8 @@ def add_new_capacities_collaborative(
     :param int scenario_length: number of hours in new scenario.
     :param float/None solar_fraction: how much new capacity should be solar.
         If given None, maintain previous ratio.
-    :param dict/None addl_curtailment: how much new curtailment is expected, by resource.
-        If given None, assume zero.
+    :param dict/None addl_curtailment: how much new curtailment is expected, by
+        resource. If given None, assume zero.
     :return: (*pandas.DataFrame*) -- targets dataframe with next capacities added.
     """
     targets = input_targets.copy()
@@ -463,7 +461,7 @@ def create_change_table(input_targets, ref_scenario):
         prev_wind = input_targets.loc[region, "wind.prev_capacity"]
         next_solar = input_targets.loc[region, "solar.next_capacity"]
         next_wind = input_targets.loc[region, "wind.next_capacity"]
-        zone_names = area_to_loadzone(ref_scenario.info["grid_model"], region)
+        zone_names = base_grid.model_immutables.area_to_loadzone(region)
         zone_ids = [base_grid.zone2id[n] for n in zone_names if n in grid_zones]
         if prev_solar > 0:
             scale = next_solar / prev_solar

powersimdata/design/generation/cost_curves.py

@@ -4,7 +4,6 @@
 import pandas as pd
 
 from powersimdata.input.grid import Grid
-from powersimdata.network.model import area_to_loadzone
 from powersimdata.utility.helpers import _check_import
 
 
@@ -195,9 +194,8 @@ def build_supply_curve(grid, num_segments, area, gen_type, area_type=None, plot=
     :param str area: Either the load zone, state name, state abbreviation, or
         interconnect.
     :param str/iterable gen_type: Generation type(s).
-    :param str area_type: one of: *'loadzone'*, *'state'*, *'state_abbr'*,
-        *'interconnect'*. Defaults to None, which allows
-        :func:`powersimdata.network.model.area_to_loadzone` to infer the type.
+    :param str area_type: one of *'loadzone'*, *'state'*, *'state_abbr'*,
+        *'interconnect'*. If set to None, type will be inferred.
     :param bool plot: If True, the supply curve plot is shown. If False, the plot is
         not shown.
     :return: (*tuple*) -- First element is a list of capacity (MW) amounts needed
@@ -232,7 +230,7 @@ def build_supply_curve(grid, num_segments, area, gen_type, area_type=None, plot=
         raise ValueError(f"{gen_type} contains invalid generation type.")
 
     # Identify the load zones that correspond to the specified area and area_type
-    returned_zones = area_to_loadzone(grid.grid_model, area, area_type)
+    returned_zones = grid.model_immutables.area_to_loadzone(area, area_type=area_type)
 
     # Trim the DataFrame to only be of the desired area and generation type
     supply_data = supply_data.loc[supply_data.zone_name.isin(returned_zones)]
@@ -427,8 +425,7 @@ def plot_linear_vs_quadratic_terms(
         interconnect.
     :param str gen_type: Generation type.
     :param str area_type: one of: *'loadzone'*, *'state'*, *'state_abbr'*,
-        *'interconnect'*. Defaults to None, which allows
-        :func:`powersimdata.network.model.area_to_loadzone` to infer the type.
+        *'interconnect'*. if set to None, the type will be inferred.
     :param bool plot: If True, the linear term vs. quadratic term plot is shown. If
         False, the plot is not shown.
     :param bool zoom: If True, filters out quadratic term outliers to enable better
@@ -474,7 +471,7 @@ def plot_linear_vs_quadratic_terms(
         raise ValueError(f"{gen_type} is not a valid generation type.")
 
     # Identify the load zones that correspond to the specified area and area_type
-    returned_zones = area_to_loadzone(grid.grid_model, area, area_type)
+    returned_zones = grid.model_immutables.area_to_loadzone(area, area_type=area_type)
 
     # Trim the DataFrame to only be of the desired area and generation type
     supply_data = supply_data.loc[supply_data.zone_name.isin(returned_zones)]
@@ -549,8 +546,7 @@ def plot_capacity_vs_price(
         interconnect.
     :param str gen_type: Generation type.
     :param str area_type: one of: *'loadzone'*, *'state'*, *'state_abbr'*,
-        *'interconnect'*. Defaults to None, which allows
-        :func:`powersimdata.network.model.area_to_loadzone` to infer the type.
+        *'interconnect'*. If set to None, the type will be inferred.
     :param bool plot: If True, the supply curve plot is shown. If False, the plot is
         not shown.
     :return: (*None*) -- The capacity vs. price plot is displayed according to the user.
@@ -581,7 +577,7 @@ def plot_capacity_vs_price(
         raise ValueError(f"{gen_type} is not a valid generation type.")
 
     # Identify the load zones that correspond to the specified area and area_type
-    returned_zones = area_to_loadzone(grid.grid_model, area, area_type)
+    returned_zones = grid.model_immutables.area_to_loadzone(area, area_type=area_type)
 
     # Trim the DataFrame to only be of the desired area and generation type
     supply_data = supply_data.loc[supply_data.zone_name.isin(returned_zones)]

powersimdata/design/transmission/upgrade.py

@@ -2,7 +2,6 @@
 
 from powersimdata.design.investment.investment_costs import _calculate_ac_inv_costs
 from powersimdata.input.grid import Grid
-from powersimdata.network.model import area_to_loadzone
 from powersimdata.utility.distance import haversine
 
 
@@ -151,7 +150,7 @@ def get_branches_by_area(grid, area_names, method="either"):
 
     :param powersimdata.input.grid.Grid grid: Grid to query for topology.
     :param list/set/tuple area_names: an iterable of area names, used to look
-        up zone names via :func:`powersimdata.network.model.area_to_loadzone`.
+        up zone names.
     :param str method: whether to include branches which span zones. Options:
         - 'internal': only select branches which are to/from the same area.
         - 'bridging': only select branches which connect area to another.
@@ -177,7 +176,7 @@ def get_branches_by_area(grid, area_names, method="either"):
     branch = grid.branch
     selected_branches = set()
     for a in area_names:
-        load_zone_names = area_to_loadzone(grid.grid_model, a)
+        load_zone_names = grid.model_immutables.area_to_loadzone(a)
         to_bus_in_area = branch.to_zone_name.isin(load_zone_names)
         from_bus_in_area = branch.from_zone_name.isin(load_zone_names)
         if method in ("internal", "either"):