From bcf3bedf0057813725cf5ac3edc2594a65ae153f Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 2 Sep 2022 15:54:01 -0700
Subject: [PATCH 01/17] feat: make FromPyPSA object a Grid object

---
 powersimdata/input/converter/pypsa_to_grid.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/powersimdata/input/converter/pypsa_to_grid.py b/powersimdata/input/converter/pypsa_to_grid.py
index 4b2340842..78f1c0bb5 100644
--- a/powersimdata/input/converter/pypsa_to_grid.py
+++ b/powersimdata/input/converter/pypsa_to_grid.py
@@ -6,6 +6,7 @@
 from powersimdata.input.abstract_grid import AbstractGrid
 from powersimdata.input.const import grid_const
 from powersimdata.input.const.pypsa_const import pypsa_const
+from powersimdata.input.grid import Grid
 from powersimdata.network.constants.carrier.storage import storage as storage_const
 
 
@@ -446,3 +447,8 @@ def _translate_pnl(self, pnl, key):
             {v: pnl[k].iloc[0] for k, v in translators.items() if k in pnl}, axis=1
         )
         return df
+
+    @property
+    def __class__(self):
+        """If anyone asks, I'm a Grid object!"""
+        return Grid

From 9ff3873d81f1bb57ea735b57440e4fddb446d3c5 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 2 Sep 2022 15:57:43 -0700
Subject: [PATCH 02/17] fix: enable roundtrip conversion (#678 and #685)

---
 powersimdata/input/const/pypsa_const.py       |  54 +++++++-
 powersimdata/input/converter/pypsa_to_grid.py | 118 +++++++++---------
 .../converter/tests/test_pypsa_to_grid.py     |  18 ++-
 .../input/exporter/export_to_pypsa.py         |  26 +++-
 4 files changed, 150 insertions(+), 66 deletions(-)

diff --git a/powersimdata/input/const/pypsa_const.py b/powersimdata/input/const/pypsa_const.py
index 024df5608..b5ed4a42a 100644
--- a/powersimdata/input/const/pypsa_const.py
+++ b/powersimdata/input/const/pypsa_const.py
@@ -124,18 +124,66 @@
     "storage_gen": {
         "rename": {
             "bus_id": "bus",
-            "Pg": "p",
-            "Qg": "q",
         },
+        "default_drop_cols": [
+            "GenFuelCost",
+            "GenIOB",
+            "GenIOC",
+            "GenIOD",
+            "Pc1",
+            "Pc2",
+            "Pg",
+            "Pmax",
+            "Pmin",
+            "Qc1max",
+            "Qc1min",
+            "Qc2max",
+            "Qc2min",
+            "Qg",
+            "Qmax",
+            "Qmin",
+            "Vg",
+            "apf",
+            "interconnect",
+            "lat",
+            "lon",
+            "mBase",
+            "mu_Pmax",
+            "mu_Pmin",
+            "mu_Qmax",
+            "mu_Qmin",
+            "ramp_10",
+            "ramp_30",
+            "ramp_agc",
+            "ramp_q",
+            "status",
+            "zone_id",
+            "zone_name",
+        ],
     },
     "storage_gencost": {
-        "rename": {"c1": "marginal_cost"},
+        "rename": {"c1": "marginal_cost", "type": "cost_type"},
+        "default_drop_cols": ["c0", "c2", "interconnect", "n", "shutdown", "startup"],
     },
     "storage_storagedata": {
         "rename": {
             "OutEff": "efficiency_dispatch",
             "InEff": "efficiency_store",
             "LossFactor": "standing_loss",
+            "duration": "max_hours",
+            "genfuel": "carrier",
         },
+        "default_drop_cols": [
+            "ExpectedTerminalStorageMax",
+            "ExpectedTerminalStorageMin",
+            "InitialStorageCost",
+            "InitialStorageLowerBound",
+            "InitialStorageUpperBound",
+            "MaxStorageLevel",
+            "MinStorageLevel",
+            "TerminalStoragePrice",
+            "UnitIdx",
+            "rho",
+        ],
     },
 }
diff --git a/powersimdata/input/converter/pypsa_to_grid.py b/powersimdata/input/converter/pypsa_to_grid.py
index 78f1c0bb5..815607683 100644
--- a/powersimdata/input/converter/pypsa_to_grid.py
+++ b/powersimdata/input/converter/pypsa_to_grid.py
@@ -102,8 +102,13 @@ def _get_storage_gencost(n, storage_type):
             "Inapplicable storage_type passed to function _get_storage_gencost."
         )
 
+    # There are "type" columns in gen and gencost with "type" column reserved
+    # for gen dataframe, hence drop it here before renaming
+    df_gencost = df_gencost.drop(columns="type", errors="ignore")
     storage_gencost = _translate_df(df_gencost, "storage_gencost")
     storage_gencost.assign(type=2, n=3, c0=0, c2=0)
+    if "type" in storage_gencost:
+        storage_gencost["type"] = pd.to_numeric(storage_gencost.type, errors="ignore")
 
     return storage_gencost
 
@@ -131,6 +136,8 @@ def _get_storage_gen(n, storage_type):
     storage_gen["Vg"] = 1
     storage_gen["mBase"] = 100
     storage_gen["status"] = 1
+    storage_gen["bus_id"] = pd.to_numeric(storage_gen.bus_id, errors="ignore")
+    storage_gen["type"] = pd.to_numeric(storage_gen.type, errors="ignore")
 
     return storage_gen
 
@@ -165,27 +172,28 @@ def _read_network(self, n, add_pypsa_cols=True):
             data_loc = "pypsa"
 
         # Read in data from PyPSA
-        bus_in_pypsa = n.buses
-        sub_in_pypsa = pd.DataFrame()
-        bus2sub_in_pypsa = pd.DataFrame()
+        bus_pypsa = n.buses
+        sub_pypsa = pd.DataFrame()
+        bus2sub_pypsa = pd.DataFrame()
         gencost_cols = ["start_up_cost", "shut_down_cost", "marginal_cost"]
-        gencost_in_pypsa = n.generators[gencost_cols]
-        plant_in_pypsa = n.generators.drop(gencost_cols, axis=1)
-        lines_in_pypsa = n.lines
-        transformers_in_pypsa = n.transformers
-        branch_in_pypsa = pd.concat(
-            [lines_in_pypsa, transformers_in_pypsa],
+        gencost_pypsa = n.generators[gencost_cols]
+        plant_pypsa = n.generators.drop(gencost_cols, axis=1)
+        lines_pypsa = n.lines
+        transformers_pypsa = n.transformers
+        branch_pypsa = pd.concat(
+            [lines_pypsa, transformers_pypsa],
             join="outer",
         )
-        dcline_in_pypsa = n.links[lambda df: df.index.str[:3] != "sub"]
-        storageunits_in_pypsa = n.storage_units
-        stores_in_pypsa = n.stores
+        dcline_pypsa = n.links[lambda df: df.index.str[:3] != "sub"]
+        storageunits_pypsa = n.storage_units
+        stores_pypsa = n.stores
 
         # bus
-        df = bus_in_pypsa.drop(columns="type")
+        df = bus_pypsa.drop(columns="type")
         bus = _translate_df(df, "bus")
-        bus.type.replace(["PQ", "PV", "Slack", ""], [1, 2, 3, 4], inplace=True)
-        bus["bus_id"] = bus.index
+        bus["type"] = bus.type.replace(
+            ["(?i)PQ", "(?i)PV", "(?i)Slack", ""], [1, 2, 3, 4], regex=True
+        ).astype(int)
 
         # zones mapping
         # only relevant if the PyPSA network was originally created from PSD
@@ -205,28 +213,28 @@ def _read_network(self, n, add_pypsa_cols=True):
             sub_cols = ["name", "interconnect_sub_id", "lat", "lon", "interconnect"]
             sub = bus[bus.is_substation][sub_cols]
             sub.index = sub[sub.index.str.startswith("sub")].index.str[3:]
-            sub_in_pypsa_cols = [
+            sub_pypsa_cols = [
                 "name",
                 "interconnect_sub_id",
                 "y",
                 "x",
                 "interconnect",
             ]
-            sub_in_pypsa = bus_in_pypsa[bus_in_pypsa.is_substation][sub_in_pypsa_cols]
-            sub_in_pypsa.index = sub_in_pypsa[
-                sub_in_pypsa.index.str.startswith("sub")
+            sub_pypsa = bus_pypsa[bus_pypsa.is_substation][sub_pypsa_cols]
+            sub_pypsa.index = sub_pypsa[
+                sub_pypsa.index.str.startswith("sub")
             ].index.str[3:]
 
             bus = bus[~bus.is_substation]
-            bus_in_pypsa = bus_in_pypsa[~bus_in_pypsa.is_substation]
+            bus_pypsa = bus_pypsa[~bus_pypsa.is_substation]
 
             bus2sub = bus[["substation", "interconnect"]].copy()
             bus2sub["sub_id"] = pd.to_numeric(
                 bus2sub.pop("substation").str[3:], errors="ignore"
             )
-            bus2sub_in_pypsa = bus_in_pypsa[["substation", "interconnect"]].copy()
-            bus2sub_in_pypsa["sub_id"] = pd.to_numeric(
-                bus2sub_in_pypsa.pop("substation").str[3:], errors="ignore"
+            bus2sub_pypsa = bus_pypsa[["substation", "interconnect"]].copy()
+            bus2sub_pypsa["sub_id"] = pd.to_numeric(
+                bus2sub_pypsa.pop("substation").str[3:], errors="ignore"
             )
         else:
             warnings.warn("Substations could not be parsed.")
@@ -240,7 +248,7 @@ def _read_network(self, n, add_pypsa_cols=True):
             bus[["Bs", "Gs"]] = shunts[["Bs", "Gs"]]
 
         # plant
-        df = plant_in_pypsa.drop(columns="type")
+        df = plant_pypsa.drop(columns="type")
         plant = _translate_df(df, "generator")
         plant["ramp_30"] = n.generators["ramp_limit_up"].fillna(0)
         plant["Pmin"] *= plant["Pmax"]  # from relative to absolute value
@@ -248,20 +256,24 @@ def _read_network(self, n, add_pypsa_cols=True):
 
         # generation costs
         # for type: type of cost model (1 piecewise linear, 2 polynomial), n: number of parameters for total cost function, c(0) to c(n-1): parameters
-        gencost = _translate_df(gencost_in_pypsa, "gencost")
-        gencost = gencost.assign(type=2, n=3, c0=0, c2=0)
+        gencost = _translate_df(gencost_pypsa, "gencost")
+        gencost = gencost.assign(
+            type=2, n=3, c0=0, c2=0, interconnect=plant.get("interconnect")
+        )
 
         # branch
         # lines
         drop_cols = ["x", "r", "b", "g"]
-        df = lines_in_pypsa.drop(columns=drop_cols, errors="ignore")
+        df = lines_pypsa.drop(columns=drop_cols, errors="ignore")
         lines = _translate_df(df, "branch")
-        lines["branch_device_type"] = "Line"
+        lines["branch_device_type"] = lines.get("branch_device_type", "Line")
 
         # transformers
-        df = transformers_in_pypsa.drop(columns=drop_cols, errors="ignore")
+        df = transformers_pypsa.drop(columns=drop_cols, errors="ignore")
         transformers = _translate_df(df, "branch")
-        transformers["branch_device_type"] = "Transformer"
+        transformers["branch_device_type"] = transformers.get(
+            "branch_device_type", "Transformer"
+        )
 
         branch = pd.concat([lines, transformers], join="outer")
         # BE model assumes a 100 MVA base, pypsa "assumes" a 1 MVA base
@@ -271,7 +283,7 @@ def _read_network(self, n, add_pypsa_cols=True):
         branch["to_bus_id"] = pd.to_numeric(branch.to_bus_id, errors="ignore")
 
         # DC lines
-        dcline = _translate_df(dcline_in_pypsa, "link")
+        dcline = _translate_df(dcline_pypsa, "link")
         dcline["Pmin"] *= dcline["Pmax"]  # convert relative to absolute
         dcline["from_bus_id"] = pd.to_numeric(dcline.from_bus_id, errors="ignore")
         dcline["to_bus_id"] = pd.to_numeric(dcline.to_bus_id, errors="ignore")
@@ -283,6 +295,7 @@ def _read_network(self, n, add_pypsa_cols=True):
         storage_gen_stores = _get_storage_gen(n, "stores")
         storage_gencost_stores = _get_storage_gencost(n, "stores")
         storage_storagedata_stores = _get_storage_storagedata(n, "stores")
+        storage_genfuel = list(n.storage_units.carrier) + list(n.stores.carrier)
 
         # Pull operational properties into grid object
         if len(n.snapshots) == 1:
@@ -319,33 +332,33 @@ def _read_network(self, n, add_pypsa_cols=True):
             "storage_storagedata_stores",
         ]
         values = [
-            (bus, bus_in_pypsa, grid_const.col_name_bus),
-            (sub, sub_in_pypsa, grid_const.col_name_sub),
-            (bus2sub, bus2sub_in_pypsa, grid_const.col_name_bus2sub),
-            (plant, plant_in_pypsa, grid_const.col_name_plant),
-            (gencost, gencost_in_pypsa, grid_const.col_name_gencost),
-            (branch, branch_in_pypsa, grid_const.col_name_branch),
-            (dcline, dcline_in_pypsa, grid_const.col_name_dcline),
+            (bus, bus_pypsa, grid_const.col_name_bus),
+            (sub, sub_pypsa, grid_const.col_name_sub),
+            (bus2sub, bus2sub_pypsa, grid_const.col_name_bus2sub),
+            (plant, plant_pypsa, grid_const.col_name_plant),
+            (gencost, gencost_pypsa, grid_const.col_name_gencost),
+            (branch, branch_pypsa, grid_const.col_name_branch),
+            (dcline, dcline_pypsa, grid_const.col_name_dcline),
             (
                 storage_gen_storageunits,
-                storageunits_in_pypsa,
+                storageunits_pypsa,
                 grid_const.col_name_plant,
             ),
             (
                 storage_gencost_storageunits,
-                storageunits_in_pypsa,
+                storageunits_pypsa,
                 grid_const.col_name_gencost,
             ),
             (
                 storage_storagedata_storageunits,
-                storageunits_in_pypsa,
+                storageunits_pypsa,
                 grid_const.col_name_storage_storagedata,
             ),
-            (storage_gen_stores, stores_in_pypsa, grid_const.col_name_plant),
-            (storage_gencost_stores, stores_in_pypsa, grid_const.col_name_gencost),
+            (storage_gen_stores, stores_pypsa, grid_const.col_name_plant),
+            (storage_gencost_stores, stores_pypsa, grid_const.col_name_gencost),
             (
                 storage_storagedata_stores,
-                stores_in_pypsa,
+                stores_pypsa,
                 grid_const.col_name_storage_storagedata,
             ),
         ]
@@ -353,15 +366,11 @@ def _read_network(self, n, add_pypsa_cols=True):
         for k, v in zip(keys, values):
             df_psd, df_pypsa, const_location = v
 
-            # Reindex
-            if k == "branch":
-                const_location += ["branch_device_type"]
-
             df_psd = df_psd.reindex(const_location, axis="columns")
 
             # Add renamed PyPSA columns
             if add_pypsa_cols:
-                df_pypsa = df_pypsa.add_prefix("PyPSA_")
+                df_pypsa = df_pypsa.add_prefix("pypsa_")
 
                 df_psd = pd.concat([df_psd, df_pypsa], axis=1)
 
@@ -370,25 +379,19 @@ def _read_network(self, n, add_pypsa_cols=True):
 
             data[k] = df_psd
 
-        # Append individual columns
-        if not n.shunt_impedances.empty:
-            data["bus"]["includes_pypsa_shunt"] = True
-        else:
-            data["bus"]["includes_pypsa_shunt"] = False
-
         for df in (
             data["storage_gen_storageunits"],
             data["storage_gencost_storageunits"],
             data["storage_storagedata_storageunits"],
         ):
-            df["which_storage_in_pypsa"] = "storage_units"
+            df["pypsa_component"] = "storage_units"
 
         for df in (
             data["storage_gen_stores"],
             data["storage_gencost_stores"],
             data["storage_storagedata_stores"],
         ):
-            df["which_storage_in_pypsa"] = "stores"
+            df["pypsa_component"] = "stores"
 
         # Build PSD grid object
         self.data_loc = data_loc
@@ -418,6 +421,7 @@ def _read_network(self, n, add_pypsa_cols=True):
             ],
             join="outer",
         )
+        self.storage["genfuel"] = storage_genfuel
         self.storage.update(storage_const)
 
         # Set index names to match PSD
diff --git a/powersimdata/input/converter/tests/test_pypsa_to_grid.py b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
index 948a121d2..3e46a6304 100644
--- a/powersimdata/input/converter/tests/test_pypsa_to_grid.py
+++ b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
@@ -39,7 +39,7 @@ def test_import_exported_network():
     ref = Grid("Western")
     kwargs = dict(add_substations=True, add_load_shedding=False, add_all_columns=True)
     n = export_to_pypsa(ref, **kwargs)
-    test = FromPyPSA(n)
+    test = FromPyPSA(n, add_pypsa_cols=False)
 
     # Only a scaled version of linear cost term is exported to pypsa
     # Test whether the exported marginal cost is in the same order of magnitude
@@ -47,8 +47,24 @@ def test_import_exported_network():
     test_total_c1 = test.gencost["before"]["c1"].sum()
     assert ref_total_c1 / test_total_c1 > 0.95 and ref_total_c1 / test_total_c1 < 1.05
 
+    # Now overwrite costs
+    for c in ["c0", "c1", "c2"]:
+        test.gencost["before"][c] = ref.gencost["before"][c]
+        test.gencost["after"][c] = ref.gencost["after"][c]
+
     # Due to rounding errors we have to compare some columns in advance
     rtol = 1e-15
     assert_series_equal(ref.branch.x, test.branch.x, rtol=rtol)
     assert_series_equal(ref.branch.r, test.branch.r, rtol=rtol)
     assert_series_equal(ref.bus.Va, test.bus.Va, rtol=rtol)
+
+    test.branch.x = ref.branch.x
+    test.branch.r = ref.branch.r
+    test.bus.Va = ref.bus.Va
+
+    # storage specification is need in import but has to removed for testing
+    test.storage["gencost"].drop(columns="pypsa_component", inplace=True)
+    test.storage["gen"].drop(columns="pypsa_component", inplace=True)
+    test.storage["StorageData"].drop(columns="pypsa_component", inplace=True)
+
+    assert ref == test
diff --git a/powersimdata/input/exporter/export_to_pypsa.py b/powersimdata/input/exporter/export_to_pypsa.py
index de40db21f..c180e42b2 100644
--- a/powersimdata/input/exporter/export_to_pypsa.py
+++ b/powersimdata/input/exporter/export_to_pypsa.py
@@ -1,5 +1,3 @@
-import warnings
-
 import numpy as np
 import pandas as pd
 
@@ -210,9 +208,26 @@ def export_to_pypsa(
     else:
         links_t = {v: links.pop(k).to_frame("now").T for k, v in link_rename_t.items()}
 
-    # TODO: add storage export
-    if not grid.storage["gen"].empty:
-        warnings.warn("The export of storages are not implemented yet.")
+    # STORAGES
+    # TODO: make distinction to pypsa stores
+    storage_data_keys = ["StorageData", "gen", "gencost"]
+    storage = []
+    defaults = {k: v for k, v in grid.storage.items() if k not in storage_data_keys}
+    for k in storage_data_keys:
+        rename = pypsa_const["storage_" + k.lower()]["rename"]
+        if not add_all_columns:
+            drop_cols = pypsa_const["storage_" + k.lower()]["default_drop_cols"]
+        df = grid.storage[k].rename(columns=rename).drop(columns=drop_cols)
+        storage.append(df)
+        defaults = {rename.get(k, k): v for k, v in defaults.items()}
+
+    storage = pd.concat(storage, axis=1)
+    storage = storage.loc[:, ~storage.columns.duplicated()]
+    for k, v in defaults.items():
+        storage[k] = storage[k].fillna(v) if k in storage else v
+
+    storage["p_nom"] = storage.get("Pmax")
+    storage["state_of_charge_initial"] = storage.pop("InitialStorage")
 
     # Import everything to a new pypsa network
     n = pypsa.Network()
@@ -226,6 +241,7 @@ def export_to_pypsa(
     n.madd("Line", lines.index, **lines, **lines_t)
     n.madd("Transformer", transformers.index, **transformers, **transformers_t)
     n.madd("Link", links.index, **links, **links_t)
+    n.madd("StorageUnit", storage.index, **storage)
 
     if add_substations:
         n.madd("Bus", substations.index, **substations)

From e99c87c5c4d5a2f08288cf5b40416d0422479b30 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 3 Jun 2022 17:20:54 -0700
Subject: [PATCH 03/17] feat: convert pypsa Network object to Grid object and
 profiles

---
 powersimdata/input/converter/pypsa_to_grid.py | 328 ++++++++++++++++++
 .../input/converter/pypsa_to_profiles.py      |  44 +++
 .../converter/tests/test_pypsa_to_grid.py     |  40 +++
 .../converter/tests/test_pypsa_to_profiles.py |  35 ++
 4 files changed, 447 insertions(+)
 create mode 100644 powersimdata/input/converter/pypsa_to_grid.py
 create mode 100644 powersimdata/input/converter/pypsa_to_profiles.py
 create mode 100644 powersimdata/input/converter/tests/test_pypsa_to_grid.py
 create mode 100644 powersimdata/input/converter/tests/test_pypsa_to_profiles.py

diff --git a/powersimdata/input/converter/pypsa_to_grid.py b/powersimdata/input/converter/pypsa_to_grid.py
new file mode 100644
index 000000000..19adaa8ce
--- /dev/null
+++ b/powersimdata/input/converter/pypsa_to_grid.py
@@ -0,0 +1,328 @@
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from powersimdata.input.abstract_grid import AbstractGrid
+from powersimdata.input.exporter.export_to_pypsa import (
+    pypsa_const as pypsa_export_const,
+)
+
+pypsa_import_const = {
+    "bus": {
+        "default_drop_cols": [
+            "interconnect_sub_id",
+            "is_substation",
+            "name",
+            "substation",
+            "unit",
+            "v_mag_pu_max",
+            "v_mag_pu_min",
+            "v_mag_pu_set",
+            "zone_name",
+            "carrier",
+            "sub_network",
+        ]
+    },
+    "sub": {
+        "default_select_cols": [
+            "name",
+            "interconnect_sub_id",
+            "lat",
+            "lon",
+            "interconnect",
+        ]
+    },
+    "generator": {
+        "drop_cols_in_advance": ["type"],
+        "default_drop_cols": [
+            "build_year",
+            "capital_cost",
+            "committable",
+            "control",
+            "down_time_before",
+            "efficiency",
+            "lifetime",
+            "marginal_cost",
+            "min_down_time",
+            "min_up_time",
+            "p_max_pu",
+            "p_nom_extendable",
+            "p_nom_max",
+            "p_nom_min",
+            "p_nom_opt",
+            "p_set",
+            "q_set",
+            "ramp_limit_down",
+            "ramp_limit_shut_down",
+            "ramp_limit_start_up",
+            "ramp_limit_up",
+            "shutdown_cost",
+            "sign",
+            "startup_cost",
+            "up_time_before",
+        ],
+    },
+    "gencost": {
+        "default_select_cols": [
+            "type",
+            "startup",
+            "shutdown",
+            "n",
+            "c2",
+            "c1",
+            "c0",
+            "interconnect",
+        ]
+    },
+    "branch": {
+        # these need to be dropped as they appear in both pypsa and powersimdata
+        # but need to be translated at the same time
+        "drop_cols_in_advance": [
+            "x",
+            "r",
+            "b",
+            "g",
+        ],
+        "default_drop_cols": [
+            "build_year",
+            "capital_cost",
+            "carrier",
+            "g",
+            "length",
+            "lifetime",
+            "model",
+            "num_parallel",
+            "phase_shift",
+            "r_pu_eff",
+            "s_max_pu",
+            "s_nom_extendable",
+            "s_nom_max",
+            "s_nom_min",
+            "s_nom_opt",
+            "sub_network",
+            "tap_position",
+            "tap_side",
+            "terrain_factor",
+            "type",
+            "v_ang_max",
+            "v_ang_min",
+            "v_nom",
+            "x_pu_eff",
+        ],
+    },
+    "link": {
+        "default_drop_cols": [
+            "build_year",
+            "capital_cost",
+            "carrier",
+            "efficiency",
+            "length",
+            "lifetime",
+            "marginal_cost",
+            "p_max_pu",
+            "p_nom_extendable",
+            "p_nom_max",
+            "p_nom_min",
+            "p_nom_opt",
+            "p_set",
+            "ramp_limit_down",
+            "ramp_limit_up",
+            "terrain_factor",
+            "type",
+        ]
+    },
+}
+
+
+class FromPyPSA(AbstractGrid):
+    """Grid builder for PyPSA network object.
+
+    :param pypsa.Network network: Network to read in.
+    :param bool drop_cols: columns to be dropped off PyPSA data frames
+    """
+
+    def __init__(self, network, drop_cols=True):
+        """Constructor"""
+        super().__init__()
+        self._read_network(network, drop_cols=drop_cols)
+
+    def _read_network(self, n, drop_cols=True):
+        """PyPSA Network reader.
+
+        :param pypsa.Network network: Network to read in.
+        :param bool drop_cols: columns to be dropped off PyPSA data frames
+        """
+
+        # Interconnect and data location
+        # relevant if the PyPSA network was originally created from powersimdata
+        interconnect = n.name.split(", ")
+        if len(interconnect) > 1:
+            data_loc = interconnect.pop(0)
+        else:
+            data_loc = "pypsa"
+
+        # bus
+        df = n.df("Bus").drop(columns="type")
+        bus = self._translate_df(df, "bus")
+        bus["type"] = bus.type.replace(["PQ", "PV", "slack", ""], [1, 2, 3, 4])
+        bus.index.name = "bus_id"
+
+        # zones mapping
+        # non-empty if the PyPSA network was originally created from powersimdata
+        if "zone_id" in n.buses and "zone_name" in n.buses:
+            uniques = ~n.buses.zone_id.duplicated() * n.buses.zone_id.notnull()
+            zone2id = (
+                n.buses[uniques].set_index("zone_name").zone_id.astype(int).to_dict()
+            )
+            id2zone = self._invert_dict(zone2id)
+        else:
+            zone2id = {}
+            id2zone = {}
+
+        # substations
+        if "is_substation" in bus:
+            cols = pypsa_import_const["sub"]["default_select_cols"]
+            sub = bus[bus.is_substation][cols]
+            sub.index = sub[sub.index.str.startswith("sub")].index.str[3:]
+            sub.index.name = "sub_id"
+            bus = bus[~bus.is_substation]
+            bus2sub = bus[["substation", "interconnect"]].copy()
+            bus2sub["sub_id"] = pd.to_numeric(
+                bus2sub.pop("substation").str[3:], errors="ignore"
+            )
+        else:
+            warnings.warn("Substations could not be parsed.")
+            sub = pd.DataFrame()
+            bus2sub = pd.DataFrame()
+
+        # shunts
+        if not n.shunt_impedances.empty:
+            shunts = self._translate_df(n.shunt_impedances, "bus")
+            bus[["Bs", "Gs"]] = shunts[["Bs", "Gs"]]
+
+        # plant
+        drop_cols = pypsa_import_const["generator"]["drop_cols_in_advance"]
+        df = n.generators.drop(columns=drop_cols)
+        plant = self._translate_df(df, "generator")
+        plant["ramp_30"] = n.generators["ramp_limit_up"].fillna(0)
+        plant["Pmin"] *= plant["Pmax"]  # from relative to absolute value
+        plant["bus_id"] = pd.to_numeric(plant.bus_id, errors="ignore")
+        plant.index.name = "plant_id"
+
+        # generation costs
+        cols = pypsa_import_const["gencost"]["default_select_cols"]
+        gencost = self._translate_df(df, "cost")
+        gencost = gencost.assign(type=2, n=3, c0=0, c2=0)
+        gencost = gencost.reindex(columns=cols)
+        gencost.index.name = "plant_id"
+
+        # branch
+        drop_cols = pypsa_import_const["branch"]["drop_cols_in_advance"]
+        df = n.lines.drop(columns=drop_cols, errors="ignore")
+        lines = self._translate_df(df, "branch")
+        lines["branch_device_type"] = "Line"
+
+        df = n.transformers.drop(columns=drop_cols, errors="ignore")
+        transformers = self._translate_df(df, "branch")
+        if "branch_device_type" not in transformers:
+            transformers["branch_device_type"] = "Transfomer"
+
+        branch = pd.concat([lines, transformers])
+        branch["x"] *= 100
+        branch["r"] *= 100
+        branch["from_bus_id"] = pd.to_numeric(branch.from_bus_id, errors="ignore")
+        branch["to_bus_id"] = pd.to_numeric(branch.to_bus_id, errors="ignore")
+        branch.index.name = "branch_id"
+
+        # DC lines
+        df = n.df("Link")[lambda df: df.index.str[:3] != "sub"]
+        dcline = self._translate_df(df, "link")
+        dcline["Pmin"] *= dcline["Pmax"]  # convert relative to absolute
+        dcline["from_bus_id"] = pd.to_numeric(dcline.from_bus_id, errors="ignore")
+        dcline["to_bus_id"] = pd.to_numeric(dcline.to_bus_id, errors="ignore")
+
+        # storages
+        if not n.storage_units.empty or not n.stores.empty:
+            warnings.warn("The export of storages are not implemented yet.")
+
+        # Drop columns if wanted
+        if drop_cols:
+            self._drop_cols(bus, "bus")
+            self._drop_cols(plant, "generator")
+            self._drop_cols(branch, "branch")
+            self._drop_cols(dcline, "link")
+
+        # Pull operational properties into grid object
+        if len(n.snapshots) == 1:
+            bus = bus.assign(**self._translate_pnl(n.pnl("Bus"), "bus"))
+            bus["Va"] = np.rad2deg(bus["Va"])
+            bus = bus.assign(**self._translate_pnl(n.pnl("Load"), "bus"))
+            plant = plant.assign(**self._translate_pnl(n.pnl("Generator"), "generator"))
+            _ = pd.concat(
+                [
+                    self._translate_pnl(n.pnl(c), "branch")
+                    for c in ["Line", "Transformer"]
+                ]
+            )
+            branch = branch.assign(**_)
+            dcline = dcline.assign(**self._translate_pnl(n.pnl("Link"), "link"))
+        else:
+            plant["status"] = n.generators_t.status.any().astype(int)
+
+        # Convert to numeric
+        for df in (bus, sub, bus2sub, gencost, plant, branch, dcline):
+            df.index = pd.to_numeric(df.index, errors="ignore")
+
+        self.data_loc = data_loc
+        self.interconnect = interconnect
+        self.bus = bus
+        self.sub = sub
+        self.bus2sub = bus2sub
+        self.branch = branch.sort_index()
+        self.dcline = dcline
+        self.zone2id = zone2id
+        self.id2zone = id2zone
+        self.plant = plant
+        self.gencost["before"] = gencost
+        self.gencost["after"] = gencost
+
+    def _drop_cols(self, df, key):
+        """Drop columns in data frame. Done inplace.
+
+        :param pandas.DataFrame df: data frame to operate on.
+        :param str key: key in the :data:`pypsa_import_const` dictionary.
+        """
+        cols = pypsa_import_const[key]["default_drop_cols"]
+        df.drop(columns=cols, inplace=True, errors="ignore")
+
+    def _translate_df(self, df, key):
+        """Rename columns of a data frame.
+
+        :param pandas.DataFrame df: data frame to operate on.
+        :param str key: key in the :data:`pypsa_import_const` dictionary.
+        """
+        translators = self._invert_dict(pypsa_export_const[key]["rename"])
+        return df.rename(columns=translators)
+
+    def _translate_pnl(self, pnl, key):
+        """Translate time-dependent data frames with one time step from pypsa to static
+        data frames.
+
+        :param str pnl: name of the time-dependent dataframe.
+        :param str key: key in the :data:`pypsa_import_const` dictionary.
+        :return: (*pandas.DataFrame*) -- the static data frame
+        """
+        translators = self._invert_dict(pypsa_export_const[key]["rename_t"])
+        df = pd.concat(
+            {v: pnl[k].iloc[0] for k, v in translators.items() if k in pnl}, axis=1
+        )
+        return df
+
+    def _invert_dict(self, d):
+        """Revert dictionary
+
+        :param dict d: dictionary to revert.
+        :return: (*dict*) -- reverted dictionary.
+        """
+        return {v: k for k, v in d.items()}
diff --git a/powersimdata/input/converter/pypsa_to_profiles.py b/powersimdata/input/converter/pypsa_to_profiles.py
new file mode 100644
index 000000000..6e9cbef0a
--- /dev/null
+++ b/powersimdata/input/converter/pypsa_to_profiles.py
@@ -0,0 +1,44 @@
+import pandas as pd
+from pypsa.descriptors import get_switchable_as_dense
+
+
+def get_pypsa_gen_profile(network, kind):
+    """Return hydro, solar or wind profile enclosed in a PyPSA network.
+
+    :param pypsa.Network network: the Network object.
+    :param str kind: either *'hydro'*, *'solar'*, *'wind'*.
+    :return: (*pandas.DataFrame*) -- profile.
+    """
+    p_max_pu = get_switchable_as_dense(network, "Generator", "p_max_pu")
+    p_max_pu.columns = pd.to_numeric(p_max_pu.columns, errors="ignore")
+    p_max_pu.columns.name = None
+    p_max_pu.index.name = "UTC"
+
+    all_gen = network.generators.copy()
+    all_gen.index = pd.to_numeric(all_gen.index, errors="ignore")
+    all_gen.index.name = None
+
+    gen = all_gen.query("@kind in carrier").index
+    return p_max_pu[gen] * all_gen.p_nom[gen]
+
+
+def get_pypsa_demand_profile(network):
+    """Return demand profile enclosed in a PyPSA network.
+
+    :param pypsa.Network network: the Network object.
+    :return: (*pandas.DataFrame*) -- profile.
+    """
+    if not network.loads_t.p.empty:
+        demand = network.loads_t.p.copy()
+    else:
+        demand = network.loads_t.p_set.copy()
+    if "zone_id" in network.buses:
+        # Assume this is a PyPSA network originally created from powersimdata
+        demand = demand.groupby(
+            network.buses.zone_id.dropna().astype(int), axis=1
+        ).sum()
+    demand.columns = pd.to_numeric(demand.columns, errors="ignore")
+    demand.columns.name = None
+    demand.index.name = "UTC"
+
+    return demand
diff --git a/powersimdata/input/converter/tests/test_pypsa_to_grid.py b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
new file mode 100644
index 000000000..7df7e8549
--- /dev/null
+++ b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
@@ -0,0 +1,40 @@
+from importlib.util import find_spec
+
+import pytest
+from pandas.testing import assert_series_equal
+
+from powersimdata.input.converter.pypsa_to_grid import FromPyPSA
+from powersimdata.input.exporter.export_to_pypsa import export_to_pypsa
+from powersimdata.input.grid import Grid
+
+
+@pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
+def test_import_arbitrary_network_from_pypsa_to_grid():
+    import pypsa
+
+    n = pypsa.examples.ac_dc_meshed()
+    grid = FromPyPSA(n)
+
+    assert not grid.bus.empty
+    assert len(n.buses) == len(grid.bus)
+
+
+@pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
+def test_import_exported_network():
+
+    ref = Grid("Western")
+    kwargs = dict(add_substations=True, add_load_shedding=False, add_all_columns=True)
+    n = export_to_pypsa(ref, **kwargs)
+    test = FromPyPSA(n)
+
+    # Only a scaled version of linear cost term is exported to pypsa
+    # Test whether the exported marginal cost is in the same order of magnitude
+    ref_total_c1 = ref.gencost["before"]["c1"].sum()
+    test_total_c1 = test.gencost["before"]["c1"].sum()
+    assert ref_total_c1 / test_total_c1 > 0.95 and ref_total_c1 / test_total_c1 < 1.05
+
+    # Due to rounding errors we have to compare some columns in advance
+    rtol = 1e-15
+    assert_series_equal(ref.branch.x, test.branch.x, rtol=rtol)
+    assert_series_equal(ref.branch.r, test.branch.r, rtol=rtol)
+    assert_series_equal(ref.bus.Va, test.bus.Va, rtol=rtol)
diff --git a/powersimdata/input/converter/tests/test_pypsa_to_profiles.py b/powersimdata/input/converter/tests/test_pypsa_to_profiles.py
new file mode 100644
index 000000000..5174cd9fc
--- /dev/null
+++ b/powersimdata/input/converter/tests/test_pypsa_to_profiles.py
@@ -0,0 +1,35 @@
+from importlib.util import find_spec
+
+import pytest
+
+from powersimdata.input.converter.pypsa_to_grid import FromPyPSA
+from powersimdata.input.converter.pypsa_to_profiles import (
+    get_pypsa_demand_profile,
+    get_pypsa_gen_profile,
+)
+
+
+@pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
+def test_extract_wind():
+    import pypsa
+
+    n = pypsa.examples.ac_dc_meshed()
+    profile = get_pypsa_gen_profile(n, "wind")
+    grid = FromPyPSA(n)
+
+    assert profile.index.name == "UTC"
+    assert (
+        grid.plant.loc[profile.columns]["Pmax"].sum() * len(profile)
+        >= profile.sum().sum()
+    )
+
+
+@pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
+def test_extract_demand():
+    import pypsa
+
+    n = pypsa.examples.ac_dc_meshed()
+    profile = get_pypsa_demand_profile(n)
+
+    assert profile.index.name == "UTC"
+    assert profile.sum().sum() >= 0

From 835e48effc10812f485ddad892928c99fb6ad359 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Sat, 10 Sep 2022 17:44:45 -0700
Subject: [PATCH 04/17] test: add test for storage

---
 .../input/converter/tests/test_pypsa_to_grid.py     | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/powersimdata/input/converter/tests/test_pypsa_to_grid.py b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
index 3e46a6304..c696c5a7c 100644
--- a/powersimdata/input/converter/tests/test_pypsa_to_grid.py
+++ b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
@@ -3,9 +3,11 @@
 import pytest
 from pandas.testing import assert_series_equal
 
+from powersimdata.input.change_table import ChangeTable
 from powersimdata.input.converter.pypsa_to_grid import FromPyPSA
 from powersimdata.input.exporter.export_to_pypsa import export_to_pypsa
 from powersimdata.input.grid import Grid
+from powersimdata.input.transform_grid import TransformGrid
 
 
 @pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
@@ -36,7 +38,16 @@ def test_import_network_including_storages_from_pypsa_to_grid():
 @pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
 def test_import_exported_network():
 
-    ref = Grid("Western")
+    grid = Grid("Western")
+    ct = ChangeTable(grid)
+    storage = [
+        {"bus_id": 2021005, "capacity": 116.0},
+        {"bus_id": 2028827, "capacity": 82.5},
+        {"bus_id": 2028060, "capacity": 82.5},
+    ]
+    ct.add_storage_capacity(storage)
+    ref = TransformGrid(grid, ct.ct).get_grid()
+
     kwargs = dict(add_substations=True, add_load_shedding=False, add_all_columns=True)
     n = export_to_pypsa(ref, **kwargs)
     test = FromPyPSA(n, add_pypsa_cols=False)

From f7d3899625c8bffff21c5bc76ab12d8929c07ce2 Mon Sep 17 00:00:00 2001
From: chrstphtrs <97453975+chrstphtrs@users.noreply.github.com>
Date: Thu, 30 Jun 2022 18:31:59 +0200
Subject: [PATCH 05/17] feat: convert PyPSA storage_units/stores to Grid
 storage_data (#657)

---
 powersimdata/input/converter/pypsa_to_grid.py | 546 +++++++++++-------
 .../converter/tests/test_pypsa_to_grid.py     |  14 +
 .../input/exporter/export_to_pypsa.py         |  35 +-
 3 files changed, 370 insertions(+), 225 deletions(-)

diff --git a/powersimdata/input/converter/pypsa_to_grid.py b/powersimdata/input/converter/pypsa_to_grid.py
index 19adaa8ce..834b46089 100644
--- a/powersimdata/input/converter/pypsa_to_grid.py
+++ b/powersimdata/input/converter/pypsa_to_grid.py
@@ -3,255 +3,284 @@
 import numpy as np
 import pandas as pd
 
+from powersimdata.input import const
 from powersimdata.input.abstract_grid import AbstractGrid
-from powersimdata.input.exporter.export_to_pypsa import (
-    pypsa_const as pypsa_export_const,
-)
-
-pypsa_import_const = {
-    "bus": {
-        "default_drop_cols": [
-            "interconnect_sub_id",
-            "is_substation",
-            "name",
-            "substation",
-            "unit",
-            "v_mag_pu_max",
-            "v_mag_pu_min",
-            "v_mag_pu_set",
-            "zone_name",
-            "carrier",
-            "sub_network",
-        ]
-    },
-    "sub": {
-        "default_select_cols": [
-            "name",
-            "interconnect_sub_id",
-            "lat",
-            "lon",
-            "interconnect",
-        ]
-    },
-    "generator": {
-        "drop_cols_in_advance": ["type"],
-        "default_drop_cols": [
-            "build_year",
-            "capital_cost",
-            "committable",
-            "control",
-            "down_time_before",
-            "efficiency",
-            "lifetime",
-            "marginal_cost",
-            "min_down_time",
-            "min_up_time",
-            "p_max_pu",
-            "p_nom_extendable",
-            "p_nom_max",
-            "p_nom_min",
-            "p_nom_opt",
-            "p_set",
-            "q_set",
-            "ramp_limit_down",
-            "ramp_limit_shut_down",
-            "ramp_limit_start_up",
-            "ramp_limit_up",
-            "shutdown_cost",
-            "sign",
-            "startup_cost",
-            "up_time_before",
-        ],
-    },
-    "gencost": {
-        "default_select_cols": [
-            "type",
-            "startup",
-            "shutdown",
-            "n",
-            "c2",
-            "c1",
-            "c0",
-            "interconnect",
-        ]
-    },
-    "branch": {
-        # these need to be dropped as they appear in both pypsa and powersimdata
-        # but need to be translated at the same time
-        "drop_cols_in_advance": [
-            "x",
-            "r",
-            "b",
-            "g",
-        ],
-        "default_drop_cols": [
-            "build_year",
-            "capital_cost",
-            "carrier",
-            "g",
-            "length",
-            "lifetime",
-            "model",
-            "num_parallel",
-            "phase_shift",
-            "r_pu_eff",
-            "s_max_pu",
-            "s_nom_extendable",
-            "s_nom_max",
-            "s_nom_min",
-            "s_nom_opt",
-            "sub_network",
-            "tap_position",
-            "tap_side",
-            "terrain_factor",
-            "type",
-            "v_ang_max",
-            "v_ang_min",
-            "v_nom",
-            "x_pu_eff",
-        ],
-    },
-    "link": {
-        "default_drop_cols": [
-            "build_year",
-            "capital_cost",
-            "carrier",
-            "efficiency",
-            "length",
-            "lifetime",
-            "marginal_cost",
-            "p_max_pu",
-            "p_nom_extendable",
-            "p_nom_max",
-            "p_nom_min",
-            "p_nom_opt",
-            "p_set",
-            "ramp_limit_down",
-            "ramp_limit_up",
-            "terrain_factor",
-            "type",
-        ]
-    },
-}
+from powersimdata.input.const.pypsa_const import pypsa_const
+from powersimdata.network.constants.carrier.storage import storage as storage_const
+
+
+def _translate_df(df, key):
+    """Rename columns of a data frame.
+
+    :param pandas.DataFrame df: data frame to operate on.
+    :param str key: key in the :data:`pypsa_const` dictionary.
+    :return: (*pandas.DataFrame*) -- data frame with translated columns.
+    """
+    translators = _invert_dict(pypsa_const[key]["rename"])
+    return df.rename(columns=translators)
+
+
+def _invert_dict(d):
+    """Revert dictionary
+
+    :param dict d: dictionary to revert.
+    :return: (*dict*) -- reverted dictionary.
+    """
+    return {v: k for k, v in d.items()}
+
+
+def _get_storage_storagedata(n, storage_type):
+    """Get storage data from PyPSA for data frame "StorageData" in PSD's
+    storage dict.
+
+    :param pypsa.Network n: PyPSA network to read in.
+    :param str storage_type: key for PyPSA storage type.
+    :return: (*pandas.DataFrame*) -- data frame with storage data.
+    """
+    if storage_type == "storage_units":
+        storage_storagedata = _translate_df(n.storage_units, "storage_storagedata")
+
+        p_nom = n.storage_units["p_nom"]
+        e_nom = p_nom * n.storage_units["max_hours"]
+        cyclic_state_of_charge = n.storage_units["cyclic_state_of_charge"]
+        state_of_charge_initial = n.storage_units["state_of_charge_initial"]
+    elif storage_type == "stores":
+        storage_storagedata = _translate_df(n.stores, "storage_storagedata")
+
+        e_nom = n.stores["e_nom"]
+        cyclic_state_of_charge = n.stores["e_cyclic"]
+        state_of_charge_initial = n.stores["e_initial"]
+
+        # Efficiencies of Store are captured in link/dcline
+        storage_storagedata["OutEff"] = 1
+        storage_storagedata["InEff"] = 1
+    else:
+        warnings.warn(
+            "Inapplicable storage_type passed to function _get_storage_storagedata."
+        )
+
+    # Initial storage: If cyclic, then fill half. If not cyclic, then apply PyPSA's state_of_charge_initial.
+    storage_storagedata["InitialStorage"] = state_of_charge_initial.where(
+        ~cyclic_state_of_charge, e_nom / 2
+    )
+    # Initial storage bounds: PSD's default is same as initial storage
+    storage_storagedata["InitialStorageLowerBound"] = storage_storagedata[
+        "InitialStorage"
+    ]
+    storage_storagedata["InitialStorageUpperBound"] = storage_storagedata[
+        "InitialStorage"
+    ]
+    # Terminal storage bounds: If cyclic, then both same as initial storage. If not cyclic, then full capacity and zero.
+    storage_storagedata["ExpectedTerminalStorageMax"] = e_nom * 1
+    storage_storagedata["ExpectedTerminalStorageMin"] = e_nom * 0
+    # Apply PSD's default relationships/assumptions for remaining columns
+    storage_storagedata["InitialStorageCost"] = storage_const["energy_value"]
+    storage_storagedata["TerminalStoragePrice"] = storage_const["energy_value"]
+    storage_storagedata["MinStorageLevel"] = e_nom * storage_const["min_stor"]
+    storage_storagedata["MaxStorageLevel"] = e_nom * storage_const["max_stor"]
+    storage_storagedata["rho"] = 1
+
+    return storage_storagedata
+
+
+def _get_storage_gencost(n, storage_type):
+    """Get storage data from PyPSA for data frame "gencost" in PSD's storage
+    dict.
+
+    :param pypsa.Network n: PyPSA network to read in.
+    :param str storage_type: key for PyPSA storage type.
+    :return: (*pandas.DataFrame*) -- data frame with storage data.
+    """
+    if storage_type == "storage_units":
+        df_gencost = n.storage_units
+    elif storage_type == "stores":
+        df_gencost = n.stores
+    else:
+        warnings.warn(
+            "Inapplicable storage_type passed to function _get_storage_gencost."
+        )
+
+    storage_gencost = _translate_df(df_gencost, "storage_gencost")
+    storage_gencost.assign(type=2, n=3, c0=0, c2=0)
+
+    return storage_gencost
+
+
+def _get_storage_gen(n, storage_type):
+    """Get storage data from PyPSA for data frame "gen" in PSD's storage dict.
+
+    :param pypsa.Network n: PyPSA network to read in.
+    :param str storage_type: key for PyPSA storage type.
+    :return: (*pandas.DataFrame*) -- data frame with storage data.
+    """
+    if storage_type == "storage_units":
+        df_gen = n.storage_units
+        p_nom = n.storage_units["p_nom"]
+    elif storage_type == "stores":
+        df_gen = n.stores
+        p_nom = np.inf
+    else:
+        warnings.warn("Inapplicable storage_type passed to function _get_storage_gen.")
+
+    storage_gen = _translate_df(df_gen, "storage_gen")
+    storage_gen["Pmax"] = +p_nom
+    storage_gen["Pmin"] = -p_nom
+    storage_gen["ramp_30"] = p_nom
+    storage_gen["Vg"] = 1
+    storage_gen["mBase"] = 100
+    storage_gen["status"] = 1
+
+    return storage_gen
 
 
 class FromPyPSA(AbstractGrid):
     """Grid builder for PyPSA network object.
 
     :param pypsa.Network network: Network to read in.
-    :param bool drop_cols: columns to be dropped off PyPSA data frames
+    :param bool add_pypsa_cols: PyPSA data frames with renamed columns appended to
+        Grid object data frames.
     """
 
-    def __init__(self, network, drop_cols=True):
+    def __init__(self, network, add_pypsa_cols=True):
         """Constructor"""
         super().__init__()
-        self._read_network(network, drop_cols=drop_cols)
+        self._read_network(network, add_pypsa_cols=add_pypsa_cols)
 
-    def _read_network(self, n, drop_cols=True):
+    def _read_network(self, n, add_pypsa_cols=True):
         """PyPSA Network reader.
 
-        :param pypsa.Network network: Network to read in.
-        :param bool drop_cols: columns to be dropped off PyPSA data frames
+        :param pypsa.Network n: PyPSA network to read in.
+        :param bool add_pypsa_cols: PyPSA data frames with renamed columns appended to
+            Grid object data frames
         """
 
         # Interconnect and data location
-        # relevant if the PyPSA network was originally created from powersimdata
+        # only relevant if the PyPSA network was originally created from PSD
         interconnect = n.name.split(", ")
         if len(interconnect) > 1:
             data_loc = interconnect.pop(0)
         else:
             data_loc = "pypsa"
 
+        # Read in data from PyPSA
+        bus_in_pypsa = n.buses
+        sub_in_pypsa = pd.DataFrame()
+        bus2sub_in_pypsa = pd.DataFrame()
+        gencost_cols = ["start_up_cost", "shut_down_cost", "marginal_cost"]
+        gencost_in_pypsa = n.generators[gencost_cols]
+        plant_in_pypsa = n.generators.drop(gencost_cols, axis=1)
+        lines_in_pypsa = n.lines
+        transformers_in_pypsa = n.transformers
+        branch_in_pypsa = pd.concat(
+            [lines_in_pypsa, transformers_in_pypsa],
+            join="outer",
+        )
+        dcline_in_pypsa = n.links[lambda df: df.index.str[:3] != "sub"]
+        storageunits_in_pypsa = n.storage_units
+        stores_in_pypsa = n.stores
+
         # bus
-        df = n.df("Bus").drop(columns="type")
-        bus = self._translate_df(df, "bus")
-        bus["type"] = bus.type.replace(["PQ", "PV", "slack", ""], [1, 2, 3, 4])
-        bus.index.name = "bus_id"
+        df = bus_in_pypsa.drop(columns="type")
+        bus = _translate_df(df, "bus")
+        bus.type.replace(["PQ", "PV", "Slack", ""], [1, 2, 3, 4], inplace=True)
+        bus["bus_id"] = bus.index
 
         # zones mapping
-        # non-empty if the PyPSA network was originally created from powersimdata
+        # only relevant if the PyPSA network was originally created from PSD
         if "zone_id" in n.buses and "zone_name" in n.buses:
             uniques = ~n.buses.zone_id.duplicated() * n.buses.zone_id.notnull()
             zone2id = (
                 n.buses[uniques].set_index("zone_name").zone_id.astype(int).to_dict()
             )
-            id2zone = self._invert_dict(zone2id)
+            id2zone = _invert_dict(zone2id)
         else:
             zone2id = {}
             id2zone = {}
 
         # substations
+        # only relevant if the PyPSA network was originally created from PSD
         if "is_substation" in bus:
-            cols = pypsa_import_const["sub"]["default_select_cols"]
-            sub = bus[bus.is_substation][cols]
+            sub_cols = ["name", "interconnect_sub_id", "lat", "lon", "interconnect"]
+            sub = bus[bus.is_substation][sub_cols]
             sub.index = sub[sub.index.str.startswith("sub")].index.str[3:]
-            sub.index.name = "sub_id"
+            sub_in_pypsa_cols = [
+                "name",
+                "interconnect_sub_id",
+                "y",
+                "x",
+                "interconnect",
+            ]
+            sub_in_pypsa = bus_in_pypsa[bus_in_pypsa.is_substation][sub_in_pypsa_cols]
+            sub_in_pypsa.index = sub_in_pypsa[
+                sub_in_pypsa.index.str.startswith("sub")
+            ].index.str[3:]
+
             bus = bus[~bus.is_substation]
+            bus_in_pypsa = bus_in_pypsa[~bus_in_pypsa.is_substation]
+
             bus2sub = bus[["substation", "interconnect"]].copy()
             bus2sub["sub_id"] = pd.to_numeric(
                 bus2sub.pop("substation").str[3:], errors="ignore"
             )
+            bus2sub_in_pypsa = bus_in_pypsa[["substation", "interconnect"]].copy()
+            bus2sub_in_pypsa["sub_id"] = pd.to_numeric(
+                bus2sub_in_pypsa.pop("substation").str[3:], errors="ignore"
+            )
         else:
             warnings.warn("Substations could not be parsed.")
             sub = pd.DataFrame()
             bus2sub = pd.DataFrame()
 
         # shunts
+        # append PyPSA's shunts information to PSD's buses data frame on columns
         if not n.shunt_impedances.empty:
-            shunts = self._translate_df(n.shunt_impedances, "bus")
+            shunts = _translate_df(n.shunt_impedances, "bus")
             bus[["Bs", "Gs"]] = shunts[["Bs", "Gs"]]
 
         # plant
-        drop_cols = pypsa_import_const["generator"]["drop_cols_in_advance"]
-        df = n.generators.drop(columns=drop_cols)
-        plant = self._translate_df(df, "generator")
+        df = plant_in_pypsa.drop(columns="type")
+        plant = _translate_df(df, "generator")
         plant["ramp_30"] = n.generators["ramp_limit_up"].fillna(0)
         plant["Pmin"] *= plant["Pmax"]  # from relative to absolute value
         plant["bus_id"] = pd.to_numeric(plant.bus_id, errors="ignore")
-        plant.index.name = "plant_id"
 
         # generation costs
-        cols = pypsa_import_const["gencost"]["default_select_cols"]
-        gencost = self._translate_df(df, "cost")
+        # for type: type of cost model (1 piecewise linear, 2 polynomial), n: number of parameters for total cost function, c(0) to c(n-1): parameters
+        gencost = _translate_df(gencost_in_pypsa, "gencost")
         gencost = gencost.assign(type=2, n=3, c0=0, c2=0)
-        gencost = gencost.reindex(columns=cols)
-        gencost.index.name = "plant_id"
 
         # branch
-        drop_cols = pypsa_import_const["branch"]["drop_cols_in_advance"]
-        df = n.lines.drop(columns=drop_cols, errors="ignore")
-        lines = self._translate_df(df, "branch")
+        # lines
+        drop_cols = ["x", "r", "b", "g"]
+        df = lines_in_pypsa.drop(columns=drop_cols, errors="ignore")
+        lines = _translate_df(df, "branch")
         lines["branch_device_type"] = "Line"
 
-        df = n.transformers.drop(columns=drop_cols, errors="ignore")
-        transformers = self._translate_df(df, "branch")
-        if "branch_device_type" not in transformers:
-            transformers["branch_device_type"] = "Transfomer"
+        # transformers
+        df = transformers_in_pypsa.drop(columns=drop_cols, errors="ignore")
+        transformers = _translate_df(df, "branch")
+        transformers["branch_device_type"] = "Transformer"
 
-        branch = pd.concat([lines, transformers])
+        branch = pd.concat([lines, transformers], join="outer")
+        # BE model assumes a 100 MVA base, pypsa "assumes" a 1 MVA base
         branch["x"] *= 100
         branch["r"] *= 100
         branch["from_bus_id"] = pd.to_numeric(branch.from_bus_id, errors="ignore")
         branch["to_bus_id"] = pd.to_numeric(branch.to_bus_id, errors="ignore")
-        branch.index.name = "branch_id"
 
         # DC lines
-        df = n.df("Link")[lambda df: df.index.str[:3] != "sub"]
-        dcline = self._translate_df(df, "link")
+        dcline = _translate_df(dcline_in_pypsa, "link")
         dcline["Pmin"] *= dcline["Pmax"]  # convert relative to absolute
         dcline["from_bus_id"] = pd.to_numeric(dcline.from_bus_id, errors="ignore")
         dcline["to_bus_id"] = pd.to_numeric(dcline.to_bus_id, errors="ignore")
 
-        # storages
-        if not n.storage_units.empty or not n.stores.empty:
-            warnings.warn("The export of storages are not implemented yet.")
-
-        # Drop columns if wanted
-        if drop_cols:
-            self._drop_cols(bus, "bus")
-            self._drop_cols(plant, "generator")
-            self._drop_cols(branch, "branch")
-            self._drop_cols(dcline, "link")
+        # storage
+        storage_gen_storageunits = _get_storage_gen(n, "storage_units")
+        storage_gencost_storageunits = _get_storage_gencost(n, "storage_units")
+        storage_storagedata_storageunits = _get_storage_storagedata(n, "storage_units")
+        storage_gen_stores = _get_storage_gen(n, "stores")
+        storage_gencost_stores = _get_storage_gencost(n, "stores")
+        storage_storagedata_stores = _get_storage_storagedata(n, "stores")
 
         # Pull operational properties into grid object
         if len(n.snapshots) == 1:
@@ -270,59 +299,144 @@ def _read_network(self, n, drop_cols=True):
         else:
             plant["status"] = n.generators_t.status.any().astype(int)
 
-        # Convert to numeric
-        for df in (bus, sub, bus2sub, gencost, plant, branch, dcline):
-            df.index = pd.to_numeric(df.index, errors="ignore")
+        # Reindex data and add PyPSA columns
+        data = dict()
+        keys = [
+            "bus",
+            "sub",
+            "bus2sub",
+            "plant",
+            "gencost",
+            "branch",
+            "dcline",
+            "storage_gen_storageunits",
+            "storage_gencost_storageunits",
+            "storage_storagedata_storageunits",
+            "storage_gen_stores",
+            "storage_gencost_stores",
+            "storage_storagedata_stores",
+        ]
+        values = [
+            (bus, bus_in_pypsa, const.col_name_bus),
+            (sub, sub_in_pypsa, const.col_name_sub),
+            (bus2sub, bus2sub_in_pypsa, const.col_name_bus2sub),
+            (plant, plant_in_pypsa, const.col_name_plant),
+            (gencost, gencost_in_pypsa, const.col_name_gencost),
+            (branch, branch_in_pypsa, const.col_name_branch),
+            (dcline, dcline_in_pypsa, const.col_name_dcline),
+            (storage_gen_storageunits, storageunits_in_pypsa, const.col_name_plant),
+            (
+                storage_gencost_storageunits,
+                storageunits_in_pypsa,
+                const.col_name_gencost,
+            ),
+            (
+                storage_storagedata_storageunits,
+                storageunits_in_pypsa,
+                const.col_name_storage_storagedata,
+            ),
+            (storage_gen_stores, stores_in_pypsa, const.col_name_plant),
+            (storage_gencost_stores, stores_in_pypsa, const.col_name_gencost),
+            (
+                storage_storagedata_stores,
+                stores_in_pypsa,
+                const.col_name_storage_storagedata,
+            ),
+        ]
 
-        self.data_loc = data_loc
-        self.interconnect = interconnect
-        self.bus = bus
-        self.sub = sub
-        self.bus2sub = bus2sub
-        self.branch = branch.sort_index()
-        self.dcline = dcline
-        self.zone2id = zone2id
-        self.id2zone = id2zone
-        self.plant = plant
-        self.gencost["before"] = gencost
-        self.gencost["after"] = gencost
+        for k, v in zip(keys, values):
+            df_psd, df_pypsa, const_location = v
 
-    def _drop_cols(self, df, key):
-        """Drop columns in data frame. Done inplace.
+            # Reindex
+            if k == "branch":
+                const_location += ["branch_device_type"]
 
-        :param pandas.DataFrame df: data frame to operate on.
-        :param str key: key in the :data:`pypsa_import_const` dictionary.
-        """
-        cols = pypsa_import_const[key]["default_drop_cols"]
-        df.drop(columns=cols, inplace=True, errors="ignore")
+            df_psd = df_psd.reindex(const_location, axis="columns")
 
-    def _translate_df(self, df, key):
-        """Rename columns of a data frame.
+            # Add renamed PyPSA columns
+            if add_pypsa_cols:
+                df_pypsa = df_pypsa.add_prefix("PyPSA_")
 
-        :param pandas.DataFrame df: data frame to operate on.
-        :param str key: key in the :data:`pypsa_import_const` dictionary.
-        """
-        translators = self._invert_dict(pypsa_export_const[key]["rename"])
-        return df.rename(columns=translators)
+                df_psd = pd.concat([df_psd, df_pypsa], axis=1)
+
+            # Convert to numeric
+            df_psd.index = pd.to_numeric(df_psd.index, errors="ignore")
+
+            data[k] = df_psd
+
+        # Append individual columns
+        if not n.shunt_impedances.empty:
+            data["bus"]["includes_pypsa_shunt"] = True
+        else:
+            data["bus"]["includes_pypsa_shunt"] = False
+
+        for df in (
+            data["storage_gen_storageunits"],
+            data["storage_gencost_storageunits"],
+            data["storage_storagedata_storageunits"],
+        ):
+            df["which_storage_in_pypsa"] = "storage_units"
+
+        for df in (
+            data["storage_gen_stores"],
+            data["storage_gencost_stores"],
+            data["storage_storagedata_stores"],
+        ):
+            df["which_storage_in_pypsa"] = "stores"
+
+        # Build PSD grid object
+        self.data_loc = data_loc
+        self.interconnect = interconnect
+        self.bus = data["bus"]
+        self.sub = data["sub"]
+        self.bus2sub = data["bus2sub"]
+        self.branch = data["branch"].sort_index()
+        self.dcline = data["dcline"]
+        self.zone2id = zone2id
+        self.id2zone = id2zone
+        self.plant = data["plant"]
+        self.gencost["before"] = data["gencost"]
+        self.gencost["after"] = data["gencost"]
+        self.storage["gen"] = pd.concat(
+            [data["storage_gen_storageunits"], data["storage_gen_stores"]],
+            join="outer",
+        )
+        self.storage["gencost"] = pd.concat(
+            [data["storage_gencost_storageunits"], data["storage_gencost_stores"]],
+            join="outer",
+        )
+        self.storage["StorageData"] = pd.concat(
+            [
+                data["storage_storagedata_storageunits"],
+                data["storage_storagedata_stores"],
+            ],
+            join="outer",
+        )
+        self.storage.update(storage_const)
+
+        # Set index names to match PSD
+        self.bus.index.name = "bus_id"
+        self.plant.index.name = "plant_id"
+        self.gencost["before"].index.name = "plant_id"
+        self.gencost["after"].index.name = "plant_id"
+        self.branch.index.name = "branch_id"
+        self.dcline.index.name = "dcline_id"
+        self.sub.index.name = "sub_id"
+        self.bus2sub.index.name = "bus_id"
+        self.storage["gen"].index.name = "storage_id"
+        self.storage["gencost"].index.name = "storage_id"
+        self.storage["StorageData"].index.name = "storage_id"
 
     def _translate_pnl(self, pnl, key):
-        """Translate time-dependent data frames with one time step from pypsa to static
+        """Translate time-dependent data frames with one time step from PyPSA to static
         data frames.
 
         :param str pnl: name of the time-dependent dataframe.
-        :param str key: key in the :data:`pypsa_import_const` dictionary.
-        :return: (*pandas.DataFrame*) -- the static data frame
+        :param str key: key in the :data:`pypsa_const` dictionary.
+        :return: (*pandas.DataFrame*) -- the static data frame.
         """
-        translators = self._invert_dict(pypsa_export_const[key]["rename_t"])
+        translators = _invert_dict(pypsa_const[key]["rename_t"])
         df = pd.concat(
             {v: pnl[k].iloc[0] for k, v in translators.items() if k in pnl}, axis=1
         )
         return df
-
-    def _invert_dict(self, d):
-        """Revert dictionary
-
-        :param dict d: dictionary to revert.
-        :return: (*dict*) -- reverted dictionary.
-        """
-        return {v: k for k, v in d.items()}
diff --git a/powersimdata/input/converter/tests/test_pypsa_to_grid.py b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
index 7df7e8549..948a121d2 100644
--- a/powersimdata/input/converter/tests/test_pypsa_to_grid.py
+++ b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
@@ -19,6 +19,20 @@ def test_import_arbitrary_network_from_pypsa_to_grid():
     assert len(n.buses) == len(grid.bus)
 
 
+@pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
+def test_import_network_including_storages_from_pypsa_to_grid():
+    import pypsa
+
+    n = pypsa.examples.storage_hvdc()
+    grid = FromPyPSA(n)
+
+    assert not grid.bus.empty
+    assert len(n.buses) == len(grid.bus)
+    assert not grid.storage["gen"].empty
+    assert not grid.storage["gencost"].empty
+    assert not grid.storage["StorageData"].empty
+
+
 @pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
 def test_import_exported_network():
 
diff --git a/powersimdata/input/exporter/export_to_pypsa.py b/powersimdata/input/exporter/export_to_pypsa.py
index a8dfbb3bf..75a15768d 100644
--- a/powersimdata/input/exporter/export_to_pypsa.py
+++ b/powersimdata/input/exporter/export_to_pypsa.py
@@ -38,9 +38,9 @@
             "bus_id": "bus",
             "Pmax": "p_nom",
             "Pmin": "p_min_pu",
-            "startup_cost": "start_up_cost",
-            "shutdown_cost": "shut_down_cost",
-            "ramp_30": "ramp_limit",
+            "startup_cost": "start_up_cost",  # not used here nor in pypsa_to_grid;
+            "shutdown_cost": "shut_down_cost",  # not used here nor in pypsa_to_grid
+            "ramp_30": "ramp_limit",  # in pypsa_to_grid: ramp_limit_up
             "type": "carrier",
         },
         "rename_t": {
@@ -66,10 +66,10 @@
             "GenIOD",
         ],
     },
-    "cost": {
+    "gencost": {
         "rename": {
-            "startup": "startup_cost",
-            "shutdown": "shutdown_cost",
+            "startup": "start_up_cost",
+            "shutdown": "shut_down_cost",
             "c1": "marginal_cost",
         }
     },
@@ -81,7 +81,7 @@
             "ratio": "tap_ratio",
             "x": "x_pu",
             "r": "r_pu",
-            "g": "g_pu",
+            "g": "g_pu",  # not used in pypsa_to_grid
             "b": "b_pu",
         },
         "rename_t": {
@@ -130,6 +130,23 @@
             "muQmaxT",
         ],
     },
+    "storage_gen": {
+        "rename": {
+            "bus_id": "bus",
+            "Pg": "p",
+            "Qg": "q",
+        },
+    },
+    "storage_gencost": {
+        "rename": {"c1": "marginal_cost"},
+    },
+    "storage_storagedata": {
+        "rename": {
+            "OutEff": "efficiency_dispatch",
+            "InEff": "efficiency_store",
+            "LossFactor": "standing_loss",
+        },
+    },
 }
 
 
@@ -245,8 +262,8 @@ def export_to_pypsa(
         grid.plant.loc[~fixed, "Pmax"] + grid.plant.loc[~fixed, "Pmin"]
     )
     gencost["c1"] = linearized.combine_first(gencost["c1"])
-    gencost = gencost.rename(columns=pypsa_const["cost"]["rename"])
-    gencost = gencost[pypsa_const["cost"]["rename"].values()]
+    gencost = gencost.rename(columns=pypsa_const["gencost"]["rename"])
+    gencost = gencost[pypsa_const["gencost"]["rename"].values()]
 
     carriers = pd.DataFrame(index=generators.carrier.unique(), dtype=object)
 

From 7b2de36ba31f130ef3f35631007228b889fd6a81 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 16 Sep 2022 00:19:54 -0700
Subject: [PATCH 06/17] feat: extract profiles from pypsa network

---
 .../input/converter/pypsa_to_profiles.py      | 79 +++++++++++++++----
 1 file changed, 63 insertions(+), 16 deletions(-)

diff --git a/powersimdata/input/converter/pypsa_to_profiles.py b/powersimdata/input/converter/pypsa_to_profiles.py
index 6e9cbef0a..6a5f783a0 100644
--- a/powersimdata/input/converter/pypsa_to_profiles.py
+++ b/powersimdata/input/converter/pypsa_to_profiles.py
@@ -1,33 +1,81 @@
+import numpy as np
 import pandas as pd
-from pypsa.descriptors import get_switchable_as_dense
+import pypsa
 
 
-def get_pypsa_gen_profile(network, kind):
+def get_pypsa_gen_profile(network, profile2carrier):
     """Return hydro, solar or wind profile enclosed in a PyPSA network.
 
     :param pypsa.Network network: the Network object.
-    :param str kind: either *'hydro'*, *'solar'*, *'wind'*.
-    :return: (*pandas.DataFrame*) -- profile.
+    :param dict profile2carrier: a dictionary mapping profile type to carrier type.
+        *'hydro'*, *'solar'* and *'wind'* are valid keys. Values is a corresponding
+        set of carriers as found in the Network object.
+    :return: (*dict*) -- keys are the same ones than in ``profile2carrier``. Values
+        are profiles as data frame.
+    :raises TypeError:
+        if ``network`` is not a pypsa.components.Network object.
+        if ``profile2carrier`` is not a dict.
+        if values of ``profile2carrier`` are not an iterable.
+    :raises ValueError:
+        if keys of ``profile2carrier`` are invalid.
     """
-    p_max_pu = get_switchable_as_dense(network, "Generator", "p_max_pu")
-    p_max_pu.columns = pd.to_numeric(p_max_pu.columns, errors="ignore")
-    p_max_pu.columns.name = None
-    p_max_pu.index.name = "UTC"
+    if not isinstance(network, pypsa.components.Network):
+        raise TypeError("network must be a Network object")
+    if not isinstance(profile2carrier, dict):
+        raise TypeError("profile2carrier must be a dict")
+    if not all(isinstance(v, (list, set, tuple)) for v in profile2carrier.values()):
+        raise TypeError("values of profile2carrier must be an iterable")
+    if not set(profile2carrier).issubset({"hydro", "solar", "wind"}):
+        raise ValueError(
+            "keys of profile2carrier must be a subset of ['hydro', 'solar', 'wind']"
+        )
 
-    all_gen = network.generators.copy()
-    all_gen.index = pd.to_numeric(all_gen.index, errors="ignore")
-    all_gen.index.name = None
+    profile = {}
+    for p, c in profile2carrier.items():
+        profile[p] = pd.DataFrame()
+        for component in ["generators", "storage_units", "stores"]:
+            if hasattr(network, component):
+                carrier_in_component = getattr(network, component)
+                id_carrier = set(carrier_in_component.query("carrier==list(@c)").index)
+                for t in ["inflow", "p_max_pu"]:
+                    if t in getattr(network, component + "_t"):
+                        ts = getattr(network, component + "_t")[t]
+                        if not ts.empty:
+                            id_ts = set(ts.columns)
+                            idx = list(id_carrier.intersection(id_ts))
+                            norm = (
+                                (
+                                    ts[idx]
+                                    .max()
+                                    .combine(
+                                        carrier_in_component.loc[idx, "p_nom"],
+                                        np.maximum,
+                                    )
+                                )
+                                if t == "inflow"
+                                else 1
+                            )
+                            profile[p] = pd.concat([profile[p], ts[idx] / norm], axis=1)
+                if len(set(c) - set(carrier_in_component.carrier.unique())):
+                    continue
+                else:
+                    break
+        profile[p].rename_axis(index="UTC", columns=None, inplace=True)
 
-    gen = all_gen.query("@kind in carrier").index
-    return p_max_pu[gen] * all_gen.p_nom[gen]
+    return profile
 
 
 def get_pypsa_demand_profile(network):
     """Return demand profile enclosed in a PyPSA network.
 
     :param pypsa.Network network: the Network object.
-    :return: (*pandas.DataFrame*) -- profile.
+    :return: (*pandas.DataFrame*) -- demand profile.
+    :raises TypeError:
+        if ``network`` is not a pypsa.components.Network object.
     """
+    if not isinstance(network, pypsa.components.Network):
+        raise TypeError("network must be a Network object")
+
     if not network.loads_t.p.empty:
         demand = network.loads_t.p.copy()
     else:
@@ -38,7 +86,6 @@ def get_pypsa_demand_profile(network):
             network.buses.zone_id.dropna().astype(int), axis=1
         ).sum()
     demand.columns = pd.to_numeric(demand.columns, errors="ignore")
-    demand.columns.name = None
-    demand.index.name = "UTC"
+    demand.rename_axis(index="UTC", columns=None, inplace=True)
 
     return demand

From c1347252828c812a0fb9fa76d9bb8cd0a50ddb03 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Mon, 26 Sep 2022 11:47:37 -0700
Subject: [PATCH 07/17] docs: format docstring and remove note

---
 .../input/exporter/export_to_pypsa.py         | 41 +++++++++----------
 1 file changed, 19 insertions(+), 22 deletions(-)

diff --git a/powersimdata/input/exporter/export_to_pypsa.py b/powersimdata/input/exporter/export_to_pypsa.py
index c180e42b2..ea1767788 100644
--- a/powersimdata/input/exporter/export_to_pypsa.py
+++ b/powersimdata/input/exporter/export_to_pypsa.py
@@ -15,28 +15,25 @@ def export_to_pypsa(
 ):
     """Export a Scenario/Grid instance to a PyPSA network.
 
-    .. note::
-        This function does not export storages yet.
-
-    :param powersimdata.scenario.scenario.Scenario /\
-        powersimdata.input.grid.Grid scenario_or_grid: input object. If a Grid instance
-        is passed, operational values will be used for the single snapshot "now".
-        If a Scenario instance is passed, all available time-series will be
-        imported.
-    :param bool add_all_columns: whether to add all columns of the
-        corresponding component. If true, this will also import columns
-        that PyPSA does not process. The default is False.
-    :param bool add_substations: whether to export substations. If set
-        to True, artificial links of infinite capacity are added from each bus
-        to its substation. This is necessary as the substations are imported
-        as regualar buses in pypsa and thus require a connection to the network.
-        If set to False, the substations will not be exported. This is
-        helpful when there are no branches or dclinks connecting the
-        substations. Note that the voltage level of the substation buses is set
-        to the first bus connected to that substation. The default is False.
-    :param bool add_load_shedding: whether to add artificial load shedding
-        generators to the exported pypsa network. This ensures feasibility when
-        optimizing the exported pypsa network as is. The default is True.
+    :param powersimdata.scenario.scenario.Scenario/powersimdata.input.grid.Grid
+        scenario_or_grid: input object. If a Grid instance is passed, operational
+        values will be used for the single snapshot "now". If a Scenario instance is
+        passed, all available time-series will be imported.
+    :param bool add_all_columns: whether to add all columns of the corresponding
+        component. If true, this will also import columns that PyPSA does not process.
+        The default is False.
+    :param bool add_substations: whether to export substations. If set to True,
+        artificial links of infinite capacity are added from each bus to its
+        substation. This is necessary as the substations are imported as regualar buses in pypsa and thus require a connection to the network. If set to False, the
+        substations will not be exported. This is helpful when there are no branches or
+        dclinks connecting the substations. Note that the voltage level of the
+        substation buses is set to the first bus connected to that substation. The
+        default is False.
+    :param bool add_load_shedding: whether to add artificial load shedding generators
+        to the exported pypsa network. This ensures feasibility when optimizing the
+        exported pypsa network as is. The default is True.
+    :return: (*pypsa.components.Network*) -- the exported Network object.
+    :raises TypeError: if ``scenario_or_grid`` is not a Grid/Scenario object.
     """
     pypsa = _check_import("pypsa")
 

From 308d94676ae2a7f5b705f2212d9700370d1db329 Mon Sep 17 00:00:00 2001
From: chrstphtrs <97453975+chrstphtrs@users.noreply.github.com>
Date: Fri, 29 Jul 2022 18:08:04 +0200
Subject: [PATCH 08/17] ci: update gitignore

---
 .gitignore | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.gitignore b/.gitignore
index 617558764..c472810e8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -142,3 +142,8 @@ dmypy.json
 
 # Windows
 Thumbs.db
+
+# Misc
+environment.yaml
+.syncignore-receive*
+.syncignore-send*
\ No newline at end of file

From 9c7f6294425622bc9b8cf9e7aaad3d28d91eb8f4 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 16 Sep 2022 00:20:14 -0700
Subject: [PATCH 09/17] test: write tests for profile extraction

---
 .../converter/tests/test_pypsa_to_profiles.py | 83 +++++++++++++++----
 1 file changed, 67 insertions(+), 16 deletions(-)

diff --git a/powersimdata/input/converter/tests/test_pypsa_to_profiles.py b/powersimdata/input/converter/tests/test_pypsa_to_profiles.py
index 5174cd9fc..abbe49027 100644
--- a/powersimdata/input/converter/tests/test_pypsa_to_profiles.py
+++ b/powersimdata/input/converter/tests/test_pypsa_to_profiles.py
@@ -2,34 +2,85 @@
 
 import pytest
 
-from powersimdata.input.converter.pypsa_to_grid import FromPyPSA
 from powersimdata.input.converter.pypsa_to_profiles import (
     get_pypsa_demand_profile,
     get_pypsa_gen_profile,
 )
 
+if find_spec("pypsa"):
+    import pypsa
+
+    @pytest.fixture
+    def network():
+        return pypsa.examples.ac_dc_meshed()
+
+
+def _assert_error(err_msg, error_type, func, *args, **kwargs):
+    with pytest.raises(error_type) as excinfo:
+        func(*args, **kwargs)
+    assert err_msg in str(excinfo.value)
+
 
 @pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
-def test_extract_wind():
-    import pypsa
+def test_get_pypsa_gen_profile_argument_type(network):
+    _assert_error(
+        "network must be a Network object",
+        TypeError,
+        get_pypsa_gen_profile,
+        "network",
+        {"wind": "onwind"},
+    )
+    _assert_error(
+        "profile2carrier must be a dict",
+        TypeError,
+        get_pypsa_gen_profile,
+        network,
+        "onwind",
+    )
+    _assert_error(
+        "values of profile2carrier must be an iterable",
+        TypeError,
+        get_pypsa_gen_profile,
+        network,
+        {"solar": "PV"},
+    )
 
-    n = pypsa.examples.ac_dc_meshed()
-    profile = get_pypsa_gen_profile(n, "wind")
-    grid = FromPyPSA(n)
 
-    assert profile.index.name == "UTC"
-    assert (
-        grid.plant.loc[profile.columns]["Pmax"].sum() * len(profile)
-        >= profile.sum().sum()
+@pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
+def test_get_pypsa_gen_profile_argument_value(network):
+    _assert_error(
+        "keys of profile2carrier must be a subset of ['hydro', 'solar', 'wind']",
+        ValueError,
+        get_pypsa_gen_profile,
+        network,
+        {"offwind": ["offwind-ac", "offwind-dc"]},
     )
 
 
 @pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
-def test_extract_demand():
-    import pypsa
+def test_extract_wind(network):
+    gen_profile = get_pypsa_gen_profile(network, {"wind": ["wind"]})
+    wind_profile = gen_profile["wind"]
+
+    assert wind_profile.index.name == "UTC"
+    assert wind_profile.columns.name is None
+    assert wind_profile.sum().apply(bool).all()
+    assert wind_profile.max().max() <= 1
+
 
-    n = pypsa.examples.ac_dc_meshed()
-    profile = get_pypsa_demand_profile(n)
+def test_get_pypsa_demand_profile_argument_type():
+    _assert_error(
+        "network must be a Network object",
+        TypeError,
+        get_pypsa_demand_profile,
+        "network",
+    )
+
+
+@pytest.mark.skipif(find_spec("pypsa") is None, reason="Package PyPSA not available.")
+def test_extract_demand(network):
+    demand_profile = get_pypsa_demand_profile(network)
 
-    assert profile.index.name == "UTC"
-    assert profile.sum().sum() >= 0
+    assert demand_profile.index.name == "UTC"
+    assert demand_profile.columns.name is None
+    assert demand_profile.sum().sum() >= 0

From 9707a827fc472fbb5bca787f42fe5f2cb80b2852 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Mon, 26 Sep 2022 12:11:07 -0700
Subject: [PATCH 10/17] fix: enable grid equality for back converted pypsa
 networks (#689)

---
 powersimdata/input/const/grid_const.py        |  1 +
 powersimdata/input/const/pypsa_const.py       |  3 +-
 powersimdata/input/converter/pypsa_to_grid.py | 36 ++++++-------
 .../input/exporter/export_to_pypsa.py         | 54 ++++++++++++++-----
 4 files changed, 60 insertions(+), 34 deletions(-)

diff --git a/powersimdata/input/const/grid_const.py b/powersimdata/input/const/grid_const.py
index f9117bd9f..e099993d9 100644
--- a/powersimdata/input/const/grid_const.py
+++ b/powersimdata/input/const/grid_const.py
@@ -296,6 +296,7 @@
     "rho",
     "ExpectedTerminalStorageMax",
     "ExpectedTerminalStorageMin",
+    "duration",
 ]
 col_type_storage_storagedata = [
     "int",
diff --git a/powersimdata/input/const/pypsa_const.py b/powersimdata/input/const/pypsa_const.py
index b5ed4a42a..5667ed732 100644
--- a/powersimdata/input/const/pypsa_const.py
+++ b/powersimdata/input/const/pypsa_const.py
@@ -124,6 +124,7 @@
     "storage_gen": {
         "rename": {
             "bus_id": "bus",
+            "Pmax": "p_nom",
         },
         "default_drop_cols": [
             "GenFuelCost",
@@ -133,7 +134,6 @@
             "Pc1",
             "Pc2",
             "Pg",
-            "Pmax",
             "Pmin",
             "Qc1max",
             "Qc1min",
@@ -172,6 +172,7 @@
             "LossFactor": "standing_loss",
             "duration": "max_hours",
             "genfuel": "carrier",
+            "InitialStorage": "state_of_charge_initial",
         },
         "default_drop_cols": [
             "ExpectedTerminalStorageMax",
diff --git a/powersimdata/input/converter/pypsa_to_grid.py b/powersimdata/input/converter/pypsa_to_grid.py
index 815607683..ad8423c5c 100644
--- a/powersimdata/input/converter/pypsa_to_grid.py
+++ b/powersimdata/input/converter/pypsa_to_grid.py
@@ -40,17 +40,17 @@ def _get_storage_storagedata(n, storage_type):
     :return: (*pandas.DataFrame*) -- data frame with storage data.
     """
     if storage_type == "storage_units":
+
         storage_storagedata = _translate_df(n.storage_units, "storage_storagedata")
 
-        p_nom = n.storage_units["p_nom"]
-        e_nom = p_nom * n.storage_units["max_hours"]
-        cyclic_state_of_charge = n.storage_units["cyclic_state_of_charge"]
+        e_nom = n.storage_units.eval("p_nom * max_hours")
         state_of_charge_initial = n.storage_units["state_of_charge_initial"]
+
     elif storage_type == "stores":
+
         storage_storagedata = _translate_df(n.stores, "storage_storagedata")
 
         e_nom = n.stores["e_nom"]
-        cyclic_state_of_charge = n.stores["e_cyclic"]
         state_of_charge_initial = n.stores["e_initial"]
 
         # Efficiencies of Store are captured in link/dcline
@@ -61,27 +61,25 @@ def _get_storage_storagedata(n, storage_type):
             "Inapplicable storage_type passed to function _get_storage_storagedata."
         )
 
-    # Initial storage: If cyclic, then fill half. If not cyclic, then apply PyPSA's state_of_charge_initial.
-    storage_storagedata["InitialStorage"] = state_of_charge_initial.where(
-        ~cyclic_state_of_charge, e_nom / 2
-    )
     # Initial storage bounds: PSD's default is same as initial storage
-    storage_storagedata["InitialStorageLowerBound"] = storage_storagedata[
-        "InitialStorage"
-    ]
-    storage_storagedata["InitialStorageUpperBound"] = storage_storagedata[
-        "InitialStorage"
-    ]
-    # Terminal storage bounds: If cyclic, then both same as initial storage. If not cyclic, then full capacity and zero.
-    storage_storagedata["ExpectedTerminalStorageMax"] = e_nom * 1
-    storage_storagedata["ExpectedTerminalStorageMin"] = e_nom * 0
+    storage_storagedata["InitialStorageLowerBound"] = state_of_charge_initial
+    storage_storagedata["InitialStorageUpperBound"] = state_of_charge_initial
     # Apply PSD's default relationships/assumptions for remaining columns
     storage_storagedata["InitialStorageCost"] = storage_const["energy_value"]
     storage_storagedata["TerminalStoragePrice"] = storage_const["energy_value"]
-    storage_storagedata["MinStorageLevel"] = e_nom * storage_const["min_stor"]
-    storage_storagedata["MaxStorageLevel"] = e_nom * storage_const["max_stor"]
     storage_storagedata["rho"] = 1
 
+    # fill with heuristic defaults if non-existent
+    defaults = {
+        "MinStorageLevel": e_nom * storage_const["min_stor"],
+        "MaxStorageLevel": e_nom * storage_const["max_stor"],
+        "ExpectedTerminalStorageMax": 1,
+        "ExpectedTerminalStorageMin": 0,
+    }
+    for k, v in defaults.items():
+        if k not in storage_storagedata:
+            storage_storagedata[k] = v
+
     return storage_storagedata
 
 
diff --git a/powersimdata/input/exporter/export_to_pypsa.py b/powersimdata/input/exporter/export_to_pypsa.py
index ea1767788..fd783b6c4 100644
--- a/powersimdata/input/exporter/export_to_pypsa.py
+++ b/powersimdata/input/exporter/export_to_pypsa.py
@@ -7,6 +7,24 @@
 from powersimdata.utility.helpers import _check_import
 
 
+def restore_original_columns(df, overwrite=None):
+    """Restore original columns in data frame
+
+    :param pandas.DataFrame df: data frame to modify.
+    :param list/set/tuple overwrite: array of column(s) in ``df`` to overwrite.
+    :return: (*pandas.DataFrame*) -- data frame with original columns.
+    """
+    if not overwrite:
+        overwrite = []
+    prefix = "pypsa_"
+    for col in df.columns[df.columns.str.startswith(prefix)]:
+        target = col[len(prefix) :]
+        fallback = df.pop(col)
+        if target not in df or target in overwrite:
+            df[target] = fallback
+    return df
+
+
 def export_to_pypsa(
     scenario_or_grid,
     add_all_columns=False,
@@ -61,7 +79,7 @@ def export_to_pypsa(
             drop_cols += list(bus_rename_t)
 
     buses = grid.bus.rename(columns=bus_rename)
-    buses.control.replace([1, 2, 3, 4], ["PQ", "PV", "slack", ""], inplace=True)
+    buses.control.replace([1, 2, 3, 4], ["PQ", "PV", "Slack", ""], inplace=True)
     buses["zone_name"] = buses.zone_id.map({v: k for k, v in grid.zone2id.items()})
     buses["substation"] = "sub" + grid.bus2sub["sub_id"].astype(str)
 
@@ -72,7 +90,8 @@ def export_to_pypsa(
 
     loads = {"proportionality_factor": buses["Pd"]}
 
-    shunts = {k: buses.pop(k) for k in ["b_pu", "g_pu"]}
+    shunts = pd.DataFrame({k: buses.pop(k) for k in ["b_pu", "g_pu"]})
+    shunts = shunts.dropna(how="all")
 
     substations = grid.sub.copy().rename(columns={"lat": "y", "lon": "x"})
     substations.index = "sub" + substations.index.astype(str)
@@ -82,6 +101,7 @@ def export_to_pypsa(
     substations["v_nom"] = v_nom
 
     buses = buses.drop(columns=drop_cols, errors="ignore").sort_index(axis=1)
+    buses = restore_original_columns(buses)
 
     # now time-dependent
     if scenario:
@@ -119,17 +139,22 @@ def export_to_pypsa(
     gencost = gencost.rename(columns=pypsa_const["gencost"]["rename"])
     gencost = gencost[pypsa_const["gencost"]["rename"].values()]
 
+    generators = generators.assign(**gencost)
+    generators = restore_original_columns(generators)
+
     carriers = pd.DataFrame(index=generators.carrier.unique(), dtype=object)
 
     cars = carriers.index
-    constants = grid.model_immutables.plants
-    carriers["color"] = pd.Series(constants["type2color"]).reindex(cars)
-    carriers["nice_name"] = pd.Series(constants["type2label"]).reindex(cars)
-    carriers["co2_emissions"] = (
-        pd.Series(constants["carbon_per_mwh"]).div(1e3)
-        * pd.Series(constants["efficiency"])
-    ).reindex(cars, fill_value=0)
-    generators["efficiency"] = generators.carrier.map(constants["efficiency"]).fillna(0)
+    if grid.model_immutables is not None:
+        constants = grid.model_immutables.plants
+        carriers["color"] = pd.Series(constants["type2color"]).reindex(cars)
+        carriers["nice_name"] = pd.Series(constants["type2label"]).reindex(cars)
+        carriers["co2_emissions"] = pd.Series(constants["carbon_per_mwh"]).div(
+            1e3
+        ) * pd.Series(constants["efficiency"]).reindex(cars, fill_value=0)
+        generators["efficiency"] = generators.carrier.map(
+            constants["efficiency"]
+        ).fillna(0)
 
     # now time-dependent
     if scenario:
@@ -165,6 +190,7 @@ def export_to_pypsa(
 
     lines = branches.query("branch_device_type == 'Line'")
     lines = lines.drop(columns="branch_device_type")
+    lines = restore_original_columns(lines)
 
     transformers = branches.query(
         "branch_device_type in ['TransformerWinding', 'Transformer']"
@@ -192,6 +218,7 @@ def export_to_pypsa(
 
     links = grid.dcline.rename(columns=link_rename).drop(columns=drop_cols)
     links.p_min_pu /= links.p_nom.where(links.p_nom != 0, 1)
+    links = restore_original_columns(links, overwrite=["p_min_pu", "p_max_pu"])
 
     # SUBSTATION CONNECTORS
     sublinks = dict(
@@ -223,8 +250,7 @@ def export_to_pypsa(
     for k, v in defaults.items():
         storage[k] = storage[k].fillna(v) if k in storage else v
 
-    storage["p_nom"] = storage.get("Pmax")
-    storage["state_of_charge_initial"] = storage.pop("InitialStorage")
+    storage = restore_original_columns(storage)
 
     # Import everything to a new pypsa network
     n = pypsa.Network()
@@ -232,8 +258,8 @@ def export_to_pypsa(
         n.snapshots = loads_t["p_set"].index
     n.madd("Bus", buses.index, **buses, **buses_t)
     n.madd("Load", buses.index, bus=buses.index, **loads, **loads_t)
-    n.madd("ShuntImpedance", buses.index, bus=buses.index, **shunts)
-    n.madd("Generator", generators.index, **generators, **gencost, **generators_t)
+    n.madd("ShuntImpedance", shunts.index, bus=shunts.index, **shunts)
+    n.madd("Generator", generators.index, **generators, **generators_t)
     n.madd("Carrier", carriers.index, **carriers)
     n.madd("Line", lines.index, **lines, **lines_t)
     n.madd("Transformer", transformers.index, **transformers, **transformers_t)

From 3e103fc3e6530af0625ebe2ee703ffe90eacc16a Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 12 Aug 2022 12:00:31 -0700
Subject: [PATCH 11/17] refactor: create library of constants for grid object,
 casemat file and pypsa translators (#667)

---
 powersimdata/input/abstract_grid.py           |  41 ++-
 powersimdata/input/const/__init__.py          |   0
 .../{const.py => const/casemat_const.py}      |   0
 powersimdata/input/const/grid_const.py        | 315 ++++++++++++++++++
 powersimdata/input/const/pypsa_const.py       | 141 ++++++++
 powersimdata/input/converter/pypsa_to_grid.py |  38 ++-
 powersimdata/input/converter/reise_to_grid.py |  30 +-
 .../input/exporter/export_to_pypsa.py         | 143 +-------
 powersimdata/tests/mock_grid.py               | 147 ++------
 9 files changed, 556 insertions(+), 299 deletions(-)
 create mode 100644 powersimdata/input/const/__init__.py
 rename powersimdata/input/{const.py => const/casemat_const.py} (100%)
 create mode 100644 powersimdata/input/const/grid_const.py
 create mode 100644 powersimdata/input/const/pypsa_const.py

diff --git a/powersimdata/input/abstract_grid.py b/powersimdata/input/abstract_grid.py
index dd5de7a54..23b882ae8 100644
--- a/powersimdata/input/abstract_grid.py
+++ b/powersimdata/input/abstract_grid.py
@@ -1,6 +1,6 @@
 import pandas as pd
 
-from powersimdata.input import const
+from powersimdata.input.const import grid_const
 
 
 class AbstractGrid:
@@ -12,13 +12,32 @@ def __init__(self):
         self.interconnect = None
         self.zone2id = {}
         self.id2zone = {}
-        self.sub = pd.DataFrame()
-        self.plant = pd.DataFrame()
-        self.gencost = {"before": pd.DataFrame(), "after": pd.DataFrame()}
-        self.dcline = pd.DataFrame()
-        self.bus2sub = pd.DataFrame()
-        self.bus = pd.DataFrame()
-        self.branch = pd.DataFrame()
+        self.sub = pd.DataFrame(columns=grid_const.col_name_sub).rename_axis(
+            grid_const.indices["sub"]
+        )
+        self.plant = pd.DataFrame(columns=grid_const.col_name_plant).rename_axis(
+            grid_const.indices["plant"]
+        )
+        self.gencost = {
+            "before": pd.DataFrame(columns=grid_const.col_name_gencost).rename_axis(
+                grid_const.indices["plant"]
+            ),
+            "after": pd.DataFrame(columns=grid_const.col_name_gencost).rename_axis(
+                grid_const.indices["plant"]
+            ),
+        }
+        self.dcline = pd.DataFrame(columns=grid_const.col_name_dcline).rename_axis(
+            grid_const.indices["dcline"]
+        )
+        self.bus2sub = pd.DataFrame(columns=grid_const.col_name_bus2sub).rename_axis(
+            grid_const.indices["bus2sub"]
+        )
+        self.bus = pd.DataFrame(columns=grid_const.col_name_bus).rename_axis(
+            grid_const.indices["bus"]
+        )
+        self.branch = pd.DataFrame(columns=grid_const.col_name_branch).rename_axis(
+            grid_const.indices["branch"]
+        )
         self.storage = storage_template()
         self.grid_model = ""
         self.model_immutables = None
@@ -30,9 +49,9 @@ def storage_template():
     :return: (*dict*) -- storage structure for MATPOWER/MOST
     """
     storage = {
-        "gen": pd.DataFrame(columns=const.col_name_plant),
-        "gencost": pd.DataFrame(columns=const.col_name_gencost),
-        "StorageData": pd.DataFrame(columns=const.col_name_storage_storagedata),
+        "gen": pd.DataFrame(columns=grid_const.col_name_plant),
+        "gencost": pd.DataFrame(columns=grid_const.col_name_gencost),
+        "StorageData": pd.DataFrame(columns=grid_const.col_name_storage_storagedata),
         "genfuel": [],
         "duration": None,  # hours
         "min_stor": None,  # ratio
diff --git a/powersimdata/input/const/__init__.py b/powersimdata/input/const/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/powersimdata/input/const.py b/powersimdata/input/const/casemat_const.py
similarity index 100%
rename from powersimdata/input/const.py
rename to powersimdata/input/const/casemat_const.py
diff --git a/powersimdata/input/const/grid_const.py b/powersimdata/input/const/grid_const.py
new file mode 100644
index 000000000..f9117bd9f
--- /dev/null
+++ b/powersimdata/input/const/grid_const.py
@@ -0,0 +1,315 @@
+# The index name of each data frame
+indices = {
+    "sub": "sub_id",
+    "bus2sub": "bus_id",
+    "branch": "branch_id",
+    "bus": "bus_id",
+    "dcline": "dcline_id",
+    "plant": "plant_id",
+}
+
+# AC lines
+col_name_branch = [
+    "from_bus_id",
+    "to_bus_id",
+    "r",
+    "x",
+    "b",
+    "rateA",
+    "rateB",
+    "rateC",
+    "ratio",
+    "angle",
+    "status",
+    "angmin",
+    "angmax",
+    "Pf",
+    "Qf",
+    "Pt",
+    "Qt",
+    "mu_Sf",
+    "mu_St",
+    "mu_angmin",
+    "mu_angmax",
+    "branch_device_type",
+    "interconnect",
+    "from_zone_id",
+    "to_zone_id",
+    "from_zone_name",
+    "to_zone_name",
+    "from_lat",
+    "from_lon",
+    "to_lat",
+    "to_lon",
+]
+col_type_branch = [
+    "int",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "str",
+    "str",
+    "int",
+    "int",
+    "str",
+    "str",
+    "float",
+    "float",
+    "float",
+    "float",
+]
+
+
+# bus
+col_name_bus = [
+    "type",
+    "Pd",
+    "Qd",
+    "Gs",
+    "Bs",
+    "zone_id",
+    "Vm",
+    "Va",
+    "baseKV",
+    "loss_zone",
+    "Vmax",
+    "Vmin",
+    "lam_P",
+    "lam_Q",
+    "mu_Vmax",
+    "mu_Vmin",
+    "interconnect",
+    "lat",
+    "lon",
+]
+col_type_bus = [
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "int",
+    "float",
+    "float",
+    "float",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "str",
+    "float",
+    "float",
+]
+
+
+# bus to substations
+col_name_bus2sub = ["sub_id", "interconnect"]
+col_type_bus2sub = ["int", "str"]
+
+
+# DC lines
+col_name_dcline = [
+    "from_bus_id",
+    "to_bus_id",
+    "status",
+    "Pf",
+    "Pt",
+    "Qf",
+    "Qt",
+    "Vf",
+    "Vt",
+    "Pmin",
+    "Pmax",
+    "QminF",
+    "QmaxF",
+    "QminT",
+    "QmaxT",
+    "loss0",
+    "loss1",
+    "muPmin",
+    "muPmax",
+    "muQminF",
+    "muQmaxF",
+    "muQminT",
+    "muQmaxT",
+    "from_interconnect",
+    "to_interconnect",
+]
+col_type_dcline = [
+    "int",
+    "int",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "str",
+    "str",
+]
+
+
+# Generation Cost
+col_name_gencost = [
+    "type",
+    "startup",
+    "shutdown",
+    "n",
+    "c2",
+    "c1",
+    "c0",
+    "interconnect",
+]
+col_type_gencost = ["int", "float", "float", "int", "float", "float", "float", "str"]
+
+
+# Generator
+col_name_plant = [
+    "bus_id",
+    "Pg",
+    "Qg",
+    "Qmax",
+    "Qmin",
+    "Vg",
+    "mBase",
+    "status",
+    "Pmax",
+    "Pmin",
+    "Pc1",
+    "Pc2",
+    "Qc1min",
+    "Qc1max",
+    "Qc2min",
+    "Qc2max",
+    "ramp_agc",
+    "ramp_10",
+    "ramp_30",
+    "ramp_q",
+    "apf",
+    "mu_Pmax",
+    "mu_Pmin",
+    "mu_Qmax",
+    "mu_Qmin",
+    "type",
+    "interconnect",
+    "GenFuelCost",
+    "GenIOB",
+    "GenIOC",
+    "GenIOD",
+    "zone_id",
+    "zone_name",
+    "lat",
+    "lon",
+]
+col_type_plant = [
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "str",
+    "str",
+    "float",
+    "float",
+    "float",
+    "int",
+    "int",
+    "str",
+    "float",
+    "float",
+]
+
+
+# substations
+col_name_sub = ["name", "interconnect_sub_id", "lat", "lon", "interconnect"]
+col_type_sub = ["str", "int", "float", "float", "str"]
+
+
+# storage
+col_name_storage_storagedata = [
+    "UnitIdx",
+    "InitialStorage",
+    "InitialStorageLowerBound",
+    "InitialStorageUpperBound",
+    "InitialStorageCost",
+    "TerminalStoragePrice",
+    "MinStorageLevel",
+    "MaxStorageLevel",
+    "OutEff",
+    "InEff",
+    "LossFactor",
+    "rho",
+    "ExpectedTerminalStorageMax",
+    "ExpectedTerminalStorageMin",
+]
+col_type_storage_storagedata = [
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+]
diff --git a/powersimdata/input/const/pypsa_const.py b/powersimdata/input/const/pypsa_const.py
new file mode 100644
index 000000000..024df5608
--- /dev/null
+++ b/powersimdata/input/const/pypsa_const.py
@@ -0,0 +1,141 @@
+pypsa_const = {
+    "bus": {
+        "rename": {
+            "lat": "y",
+            "lon": "x",
+            "baseKV": "v_nom",
+            "type": "control",
+            "Gs": "g_pu",
+            "Bs": "b_pu",
+        },
+        "rename_t": {
+            "Pd": "p",
+            "Qd": "q",
+            "Vm": "v_mag_pu",
+            "Va": "v_ang",
+        },
+        "default_drop_cols": [
+            "Vmax",
+            "Vmin",
+            "lam_P",
+            "lam_Q",
+            "mu_Vmax",
+            "mu_Vmin",
+            "GenFuelCost",
+        ],
+    },
+    "generator": {
+        "rename": {
+            "bus_id": "bus",
+            "Pmax": "p_nom",
+            "Pmin": "p_min_pu",
+            "startup_cost": "start_up_cost",  # not used here nor in pypsa_to_grid;
+            "shutdown_cost": "shut_down_cost",  # not used here nor in pypsa_to_grid
+            "ramp_30": "ramp_limit",  # in pypsa_to_grid: ramp_limit_up
+            "type": "carrier",
+        },
+        "rename_t": {
+            "Pg": "p",
+            "Qg": "q",
+            "status": "status",
+        },
+        "default_drop_cols": [
+            "ramp_10",
+            "mu_Pmax",
+            "mu_Pmin",
+            "mu_Qmax",
+            "mu_Qmin",
+            "ramp_agc",
+            "Pc1",
+            "Pc2",
+            "Qc1min",
+            "Qc1max",
+            "Qc2min",
+            "Qc2max",
+            "GenIOB",
+            "GenIOC",
+            "GenIOD",
+        ],
+    },
+    "gencost": {
+        "rename": {
+            "startup": "start_up_cost",
+            "shutdown": "shut_down_cost",
+            "c1": "marginal_cost",
+        }
+    },
+    "branch": {
+        "rename": {
+            "from_bus_id": "bus0",
+            "to_bus_id": "bus1",
+            "rateA": "s_nom",
+            "ratio": "tap_ratio",
+            "x": "x_pu",
+            "r": "r_pu",
+            "g": "g_pu",  # not used in pypsa_to_grid
+            "b": "b_pu",
+        },
+        "rename_t": {
+            "Pf": "p0",
+            "Qf": "q0",
+            "Pt": "p1",
+            "Qt": "q1",
+        },
+        "default_drop_cols": [
+            "rateB",
+            "rateC",
+            "mu_St",
+            "mu_angmin",
+            "mu_angmax",
+        ],
+    },
+    "link": {
+        "rename": {
+            "from_bus_id": "bus0",
+            "to_bus_id": "bus1",
+            "rateA": "s_nom",
+            "ratio": "tap_ratio",
+            "x": "x_pu",
+            "r": "r_pu",
+            "g": "g_pu",
+            "b": "b_pu",
+            "Pmin": "p_min_pu",
+            "Pmax": "p_nom",
+        },
+        "rename_t": {
+            "Pf": "p0",
+            "Qf": "q0",
+            "Pt": "p1",
+            "Qt": "q1",
+        },
+        "default_drop_cols": [
+            "QminF",
+            "QmaxF",
+            "QminT",
+            "QmaxT",
+            "muPmin",
+            "muPmax",
+            "muQminF",
+            "muQmaxF",
+            "muQminT",
+            "muQmaxT",
+        ],
+    },
+    "storage_gen": {
+        "rename": {
+            "bus_id": "bus",
+            "Pg": "p",
+            "Qg": "q",
+        },
+    },
+    "storage_gencost": {
+        "rename": {"c1": "marginal_cost"},
+    },
+    "storage_storagedata": {
+        "rename": {
+            "OutEff": "efficiency_dispatch",
+            "InEff": "efficiency_store",
+            "LossFactor": "standing_loss",
+        },
+    },
+}
diff --git a/powersimdata/input/converter/pypsa_to_grid.py b/powersimdata/input/converter/pypsa_to_grid.py
index 834b46089..4b2340842 100644
--- a/powersimdata/input/converter/pypsa_to_grid.py
+++ b/powersimdata/input/converter/pypsa_to_grid.py
@@ -3,8 +3,8 @@
 import numpy as np
 import pandas as pd
 
-from powersimdata.input import const
 from powersimdata.input.abstract_grid import AbstractGrid
+from powersimdata.input.const import grid_const
 from powersimdata.input.const.pypsa_const import pypsa_const
 from powersimdata.network.constants.carrier.storage import storage as storage_const
 
@@ -13,7 +13,8 @@ def _translate_df(df, key):
     """Rename columns of a data frame.
 
     :param pandas.DataFrame df: data frame to operate on.
-    :param str key: key in the :data:`pypsa_const` dictionary.
+    :param str key: key in :data:`powersimdata.input.const.pypsa_const.pypsa_const`
+        dictionary.
     :return: (*pandas.DataFrame*) -- data frame with translated columns.
     """
     translators = _invert_dict(pypsa_const[key]["rename"])
@@ -317,30 +318,34 @@ def _read_network(self, n, add_pypsa_cols=True):
             "storage_storagedata_stores",
         ]
         values = [
-            (bus, bus_in_pypsa, const.col_name_bus),
-            (sub, sub_in_pypsa, const.col_name_sub),
-            (bus2sub, bus2sub_in_pypsa, const.col_name_bus2sub),
-            (plant, plant_in_pypsa, const.col_name_plant),
-            (gencost, gencost_in_pypsa, const.col_name_gencost),
-            (branch, branch_in_pypsa, const.col_name_branch),
-            (dcline, dcline_in_pypsa, const.col_name_dcline),
-            (storage_gen_storageunits, storageunits_in_pypsa, const.col_name_plant),
+            (bus, bus_in_pypsa, grid_const.col_name_bus),
+            (sub, sub_in_pypsa, grid_const.col_name_sub),
+            (bus2sub, bus2sub_in_pypsa, grid_const.col_name_bus2sub),
+            (plant, plant_in_pypsa, grid_const.col_name_plant),
+            (gencost, gencost_in_pypsa, grid_const.col_name_gencost),
+            (branch, branch_in_pypsa, grid_const.col_name_branch),
+            (dcline, dcline_in_pypsa, grid_const.col_name_dcline),
+            (
+                storage_gen_storageunits,
+                storageunits_in_pypsa,
+                grid_const.col_name_plant,
+            ),
             (
                 storage_gencost_storageunits,
                 storageunits_in_pypsa,
-                const.col_name_gencost,
+                grid_const.col_name_gencost,
             ),
             (
                 storage_storagedata_storageunits,
                 storageunits_in_pypsa,
-                const.col_name_storage_storagedata,
+                grid_const.col_name_storage_storagedata,
             ),
-            (storage_gen_stores, stores_in_pypsa, const.col_name_plant),
-            (storage_gencost_stores, stores_in_pypsa, const.col_name_gencost),
+            (storage_gen_stores, stores_in_pypsa, grid_const.col_name_plant),
+            (storage_gencost_stores, stores_in_pypsa, grid_const.col_name_gencost),
             (
                 storage_storagedata_stores,
                 stores_in_pypsa,
-                const.col_name_storage_storagedata,
+                grid_const.col_name_storage_storagedata,
             ),
         ]
 
@@ -432,7 +437,8 @@ def _translate_pnl(self, pnl, key):
         data frames.
 
         :param str pnl: name of the time-dependent dataframe.
-        :param str key: key in the :data:`pypsa_const` dictionary.
+        :param str key: key in :data:`powersimdata.input.const.pypsa_const.pypsa_const`
+            dictionary.
         :return: (*pandas.DataFrame*) -- the static data frame.
         """
         translators = _invert_dict(pypsa_const[key]["rename_t"])
diff --git a/powersimdata/input/converter/reise_to_grid.py b/powersimdata/input/converter/reise_to_grid.py
index 153fccea1..565fa7e6e 100644
--- a/powersimdata/input/converter/reise_to_grid.py
+++ b/powersimdata/input/converter/reise_to_grid.py
@@ -6,8 +6,8 @@
 
 from powersimdata.data_access.context import Context
 from powersimdata.data_access.scenario_list import ScenarioListManager
-from powersimdata.input import const
 from powersimdata.input.abstract_grid import AbstractGrid
+from powersimdata.input.const import casemat_const
 from powersimdata.input.converter.helpers import (
     add_coord_to_grid_data_frames,
     add_interconnect_to_grid_data_frames,
@@ -233,13 +233,13 @@ def column_name_provider():
     :return: (*dict*) -- dictionary of data frame columns name.
     """
     col_name = {
-        "sub": const.col_name_sub,
-        "bus": const.col_name_bus,
-        "bus2sub": const.col_name_bus2sub,
-        "branch": const.col_name_branch,
-        "dcline": const.col_name_dcline,
-        "plant": const.col_name_plant,
-        "heat_rate_curve": const.col_name_heat_rate_curve,
+        "sub": casemat_const.col_name_sub,
+        "bus": casemat_const.col_name_bus,
+        "bus2sub": casemat_const.col_name_bus2sub,
+        "branch": casemat_const.col_name_branch,
+        "dcline": casemat_const.col_name_dcline,
+        "plant": casemat_const.col_name_plant,
+        "heat_rate_curve": casemat_const.col_name_heat_rate_curve,
     }
     return col_name
 
@@ -250,13 +250,13 @@ def column_type_provider():
     :return: (*dict*) -- dictionary of data frame columns type.
     """
     col_type = {
-        "sub": const.col_type_sub,
-        "bus": const.col_type_bus,
-        "bus2sub": const.col_type_bus2sub,
-        "branch": const.col_type_branch,
-        "dcline": const.col_type_dcline,
-        "plant": const.col_type_plant,
-        "heat_rate_curve": const.col_type_heat_rate_curve,
+        "sub": casemat_const.col_type_sub,
+        "bus": casemat_const.col_type_bus,
+        "bus2sub": casemat_const.col_type_bus2sub,
+        "branch": casemat_const.col_type_branch,
+        "dcline": casemat_const.col_type_dcline,
+        "plant": casemat_const.col_type_plant,
+        "heat_rate_curve": casemat_const.col_type_heat_rate_curve,
     }
     return col_type
 
diff --git a/powersimdata/input/exporter/export_to_pypsa.py b/powersimdata/input/exporter/export_to_pypsa.py
index 75a15768d..de40db21f 100644
--- a/powersimdata/input/exporter/export_to_pypsa.py
+++ b/powersimdata/input/exporter/export_to_pypsa.py
@@ -3,152 +3,11 @@
 import numpy as np
 import pandas as pd
 
+from powersimdata.input.const.pypsa_const import pypsa_const
 from powersimdata.input.grid import Grid
 from powersimdata.scenario.scenario import Scenario
 from powersimdata.utility.helpers import _check_import
 
-pypsa_const = {
-    "bus": {
-        "rename": {
-            "lat": "y",
-            "lon": "x",
-            "baseKV": "v_nom",
-            "type": "control",
-            "Gs": "g_pu",
-            "Bs": "b_pu",
-        },
-        "rename_t": {
-            "Pd": "p",
-            "Qd": "q",
-            "Vm": "v_mag_pu",
-            "Va": "v_ang",
-        },
-        "default_drop_cols": [
-            "Vmax",
-            "Vmin",
-            "lam_P",
-            "lam_Q",
-            "mu_Vmax",
-            "mu_Vmin",
-            "GenFuelCost",
-        ],
-    },
-    "generator": {
-        "rename": {
-            "bus_id": "bus",
-            "Pmax": "p_nom",
-            "Pmin": "p_min_pu",
-            "startup_cost": "start_up_cost",  # not used here nor in pypsa_to_grid;
-            "shutdown_cost": "shut_down_cost",  # not used here nor in pypsa_to_grid
-            "ramp_30": "ramp_limit",  # in pypsa_to_grid: ramp_limit_up
-            "type": "carrier",
-        },
-        "rename_t": {
-            "Pg": "p",
-            "Qg": "q",
-            "status": "status",
-        },
-        "default_drop_cols": [
-            "ramp_10",
-            "mu_Pmax",
-            "mu_Pmin",
-            "mu_Qmax",
-            "mu_Qmin",
-            "ramp_agc",
-            "Pc1",
-            "Pc2",
-            "Qc1min",
-            "Qc1max",
-            "Qc2min",
-            "Qc2max",
-            "GenIOB",
-            "GenIOC",
-            "GenIOD",
-        ],
-    },
-    "gencost": {
-        "rename": {
-            "startup": "start_up_cost",
-            "shutdown": "shut_down_cost",
-            "c1": "marginal_cost",
-        }
-    },
-    "branch": {
-        "rename": {
-            "from_bus_id": "bus0",
-            "to_bus_id": "bus1",
-            "rateA": "s_nom",
-            "ratio": "tap_ratio",
-            "x": "x_pu",
-            "r": "r_pu",
-            "g": "g_pu",  # not used in pypsa_to_grid
-            "b": "b_pu",
-        },
-        "rename_t": {
-            "Pf": "p0",
-            "Qf": "q0",
-            "Pt": "p1",
-            "Qt": "q1",
-        },
-        "default_drop_cols": [
-            "rateB",
-            "rateC",
-            "mu_St",
-            "mu_angmin",
-            "mu_angmax",
-        ],
-    },
-    "link": {
-        "rename": {
-            "from_bus_id": "bus0",
-            "to_bus_id": "bus1",
-            "rateA": "s_nom",
-            "ratio": "tap_ratio",
-            "x": "x_pu",
-            "r": "r_pu",
-            "g": "g_pu",
-            "b": "b_pu",
-            "Pmin": "p_min_pu",
-            "Pmax": "p_nom",
-        },
-        "rename_t": {
-            "Pf": "p0",
-            "Qf": "q0",
-            "Pt": "p1",
-            "Qt": "q1",
-        },
-        "default_drop_cols": [
-            "QminF",
-            "QmaxF",
-            "QminT",
-            "QmaxT",
-            "muPmin",
-            "muPmax",
-            "muQminF",
-            "muQmaxF",
-            "muQminT",
-            "muQmaxT",
-        ],
-    },
-    "storage_gen": {
-        "rename": {
-            "bus_id": "bus",
-            "Pg": "p",
-            "Qg": "q",
-        },
-    },
-    "storage_gencost": {
-        "rename": {"c1": "marginal_cost"},
-    },
-    "storage_storagedata": {
-        "rename": {
-            "OutEff": "efficiency_dispatch",
-            "InEff": "efficiency_store",
-            "LossFactor": "standing_loss",
-        },
-    },
-}
-
 
 def export_to_pypsa(
     scenario_or_grid,
diff --git a/powersimdata/tests/mock_grid.py b/powersimdata/tests/mock_grid.py
index 4092dd012..b8b22144b 100644
--- a/powersimdata/tests/mock_grid.py
+++ b/powersimdata/tests/mock_grid.py
@@ -1,85 +1,26 @@
 import pandas as pd
 
-from powersimdata.input import const
+from powersimdata.input.abstract_grid import AbstractGrid
+from powersimdata.input.const import grid_const
 from powersimdata.input.grid import Grid
 from powersimdata.network.model import ModelImmutables
 
-# The index name of each data frame attribute
-indices = {
-    "sub": "sub_id",
-    "bus2sub": "bus_id",
-    "branch": "branch_id",
-    "bus": "bus_id",
-    "dcline": "dcline_id",
-    "plant": "plant_id",
-}
 
-gencost_names = {"gencost_before": "before", "gencost_after": "after"}
-storage_names = {"storage_gen": "gen", "storage_StorageData": "StorageData"}
-acceptable_keys = (
-    set(indices.keys()) | set(gencost_names.keys()) | set(storage_names.keys())
-)
-
-# The column names of each data frame attribute
-sub_columns = ["name", "interconnect_sub_id", "lat", "lon", "interconnect"]
-
-bus2sub_columns = ["sub_id", "interconnect"]
-
-branch_augment_columns = [
-    "branch_device_type",
-    "interconnect",
-    "from_zone_id",
-    "to_zone_id",
-    "from_zone_name",
-    "to_zone_name",
-    "from_lat",
-    "from_lon",
-    "to_lat",
-    "to_lon",
-]
-branch_columns = const.col_name_branch + branch_augment_columns
-
-bus_augment_columns = ["interconnect", "lat", "lon"]
-bus_columns = const.col_name_bus + bus_augment_columns
-
-dcline_augment_columns = ["from_interconnect", "to_interconnect"]
-dcline_columns = const.col_name_dcline + dcline_augment_columns
-
-gencost_augment_columns = ["interconnect"]
-gencost_columns = const.col_name_gencost + gencost_augment_columns
-
-plant_augment_columns = [
-    "type",
-    "interconnect",
-    "GenFuelCost",
-    "GenIOB",
-    "GenIOC",
-    "GenIOD",
-    "zone_id",
-    "zone_name",
-    "lat",
-    "lon",
-]
-plant_columns = const.col_name_plant + plant_augment_columns
-
-storage_columns = {
-    # The first 21 columns of plant are all that's necessary
-    "gen": plant_columns[:21],
-    "StorageData": const.col_name_storage_storagedata,
-}
-
-
-class MockGrid:
+class MockGrid(AbstractGrid):
     def __init__(self, grid_attrs=None, model="usa_tamu"):
         """Constructor.
 
-        :param dict grid_attrs: dictionary of {*field_name*, *data_dict*} pairs
-            where *field_name* is the name of the data frame (sub, bus2sub,
-            branch, bus, dcline, gencost or plant) and *data_dict* a dictionary
-            in which the keys are the column name of the data frame associated
-            to *field_name* and the values are a list of values.
+        :param dict grid_attrs: dictionary of {*field_name*, *data_dict*} pairs where
+            *field_name* can be: sub, bus2sub, branch, bus, dcline, plant,
+            gencost_before, gencost_after, storage_gen, storage_StorageData and
+            *data_dict* is a dictionary in which the keys are the column name of the
+            data frame associated to *field_name* and the values are a list of values.
         :param str model: grid model. Use to access geographical information such
             as loadzones, interconnections, etc.
+        :raises TypeError:
+            if ``grid_attrs`` is not a dict.
+            if keys of ``grid_attrs`` are not str.
+        :raises ValueError: if a key of ``grid_attrs`` is incorrect.
         """
         if grid_attrs is None:
             grid_attrs = {}
@@ -91,59 +32,35 @@ def __init__(self, grid_attrs=None, model="usa_tamu"):
             if not isinstance(key, str):
                 raise TypeError("grid_attrs keys must all be str")
 
-        extra_keys = set(grid_attrs.keys()) - acceptable_keys
+        extra_keys = set(grid_attrs) - set(grid_const.indices).union(
+            {"gencost_before", "gencost_after", "storage_gen", "storage_StorageData"}
+        )
         if len(extra_keys) > 0:
             raise ValueError("Got unknown key(s):" + str(extra_keys))
 
+        super().__init__()
         self.grid_model = model
         self.model_immutables = ModelImmutables(model)
 
-        cols = {
-            "sub": sub_columns,
-            "bus2sub": bus2sub_columns,
-            "branch": branch_columns,
-            "bus": bus_columns,
-            "dcline": dcline_columns,
-            "plant": plant_columns,
-        }
-
-        self.data_loc = None
-        self.interconnect = None
-        self.zone2id = {}
-        self.id2zone = {}
-
-        # Loop through names for grid data frames, add (maybe empty) data
-        # frames.
-        for df_name in indices:
-            if df_name in grid_attrs:
-                df = pd.DataFrame(grid_attrs[df_name])
+        other = {}
+        for k, v in grid_attrs.items():
+            if k in grid_const.indices:
+                setattr(self, k, pd.DataFrame(v).set_index(grid_const.indices[k]))
             else:
-                df = pd.DataFrame(columns=([indices[df_name]] + cols[df_name]))
-            df.set_index(indices[df_name], inplace=True)
-            setattr(self, df_name, df)
+                s = k.split("_")
+                df = (
+                    pd.DataFrame(v).set_index("plant_id")
+                    if s[0] == "gencost"
+                    else pd.DataFrame(v)
+                )
 
-        # Gencost is special because there are two dataframes in a dict
-        gencost = {}
-        for gridattr_name, gc_name in gencost_names.items():
-            if gridattr_name in grid_attrs:
-                df = pd.DataFrame(grid_attrs[gridattr_name])
-            else:
-                df = pd.DataFrame(columns=(["plant_id"] + gencost_columns))
-            df.set_index("plant_id", inplace=True)
-            gencost[gc_name] = df
-        self.gencost = gencost
+                if s[0] not in other:
+                    other[s[0]] = {s[1]: df}
+                else:
+                    other[s[0]][s[1]] = df
 
-        # Storage is special because there are multiple dataframes in a dict
-        storage = {}
-        for storage_attr_name, storage_name in storage_names.items():
-            if storage_attr_name in grid_attrs:
-                df = pd.DataFrame(grid_attrs[storage_attr_name])
-            else:
-                df = pd.DataFrame(
-                    columns=(["plant_id"] + storage_columns[storage_name])
-                )
-            storage[storage_name] = df
-        self.storage = storage
+        for k, v in other.items():
+            setattr(self, k, v)
 
     @property
     def __class__(self):

From 9a4d2909db0f895986a2fcb2712a1210121f2ee3 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 16 Sep 2022 14:35:53 -0700
Subject: [PATCH 12/17] feat: normalize inflow profiles by max

---
 powersimdata/input/converter/pypsa_to_profiles.py | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/powersimdata/input/converter/pypsa_to_profiles.py b/powersimdata/input/converter/pypsa_to_profiles.py
index 6a5f783a0..4ce7a2851 100644
--- a/powersimdata/input/converter/pypsa_to_profiles.py
+++ b/powersimdata/input/converter/pypsa_to_profiles.py
@@ -1,4 +1,3 @@
-import numpy as np
 import pandas as pd
 import pypsa
 
@@ -43,18 +42,7 @@ def get_pypsa_gen_profile(network, profile2carrier):
                         if not ts.empty:
                             id_ts = set(ts.columns)
                             idx = list(id_carrier.intersection(id_ts))
-                            norm = (
-                                (
-                                    ts[idx]
-                                    .max()
-                                    .combine(
-                                        carrier_in_component.loc[idx, "p_nom"],
-                                        np.maximum,
-                                    )
-                                )
-                                if t == "inflow"
-                                else 1
-                            )
+                            norm = ts[idx].max() if t == "inflow" else 1
                             profile[p] = pd.concat([profile[p], ts[idx] / norm], axis=1)
                 if len(set(c) - set(carrier_in_component.carrier.unique())):
                     continue

From 0444b31d62da01ec9f17c77d4b80866dc4f9a39d Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Tue, 27 Sep 2022 12:51:50 -0700
Subject: [PATCH 13/17] refactor: simplify logic

---
 .../input/converter/pypsa_to_profiles.py      | 28 ++++++++-----------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/powersimdata/input/converter/pypsa_to_profiles.py b/powersimdata/input/converter/pypsa_to_profiles.py
index 4ce7a2851..09ee09fe6 100644
--- a/powersimdata/input/converter/pypsa_to_profiles.py
+++ b/powersimdata/input/converter/pypsa_to_profiles.py
@@ -1,5 +1,6 @@
 import pandas as pd
 import pypsa
+from pypsa.descriptors import get_switchable_as_dense
 
 
 def get_pypsa_gen_profile(network, profile2carrier):
@@ -29,25 +30,20 @@ def get_pypsa_gen_profile(network, profile2carrier):
             "keys of profile2carrier must be a subset of ['hydro', 'solar', 'wind']"
         )
 
+    component2timeseries = {
+        "Generator": "p_max_pu",
+        "StorageUnit": "inflow",
+    }
     profile = {}
     for p, c in profile2carrier.items():
         profile[p] = pd.DataFrame()
-        for component in ["generators", "storage_units", "stores"]:
-            if hasattr(network, component):
-                carrier_in_component = getattr(network, component)
-                id_carrier = set(carrier_in_component.query("carrier==list(@c)").index)
-                for t in ["inflow", "p_max_pu"]:
-                    if t in getattr(network, component + "_t"):
-                        ts = getattr(network, component + "_t")[t]
-                        if not ts.empty:
-                            id_ts = set(ts.columns)
-                            idx = list(id_carrier.intersection(id_ts))
-                            norm = ts[idx].max() if t == "inflow" else 1
-                            profile[p] = pd.concat([profile[p], ts[idx] / norm], axis=1)
-                if len(set(c) - set(carrier_in_component.carrier.unique())):
-                    continue
-                else:
-                    break
+        for component, ts in component2timeseries.items():
+            id_carrier = network.df(component).query("carrier==list(@c)").index
+            ts_carrier = get_switchable_as_dense(network, component, ts)[id_carrier]
+            if not ts_carrier.empty:
+                norm = ts_carrier.max().replace(0, 1) if ts == "inflow" else 1
+                profile[p] = pd.concat([profile[p], ts_carrier / norm], axis=1)
+
         profile[p].rename_axis(index="UTC", columns=None, inplace=True)
 
     return profile

From b8053256a77f4cbcaf5f5026c20541d4d288bf3c Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Tue, 25 Oct 2022 21:48:57 +0200
Subject: [PATCH 14/17] feat: extract substation from arbitrary pypsa networks
 (#674)

---
 powersimdata/input/converter/pypsa_to_grid.py | 54 +++++++++++--------
 1 file changed, 32 insertions(+), 22 deletions(-)

diff --git a/powersimdata/input/converter/pypsa_to_grid.py b/powersimdata/input/converter/pypsa_to_grid.py
index ad8423c5c..a9f0eecac 100644
--- a/powersimdata/input/converter/pypsa_to_grid.py
+++ b/powersimdata/input/converter/pypsa_to_grid.py
@@ -172,7 +172,6 @@ def _read_network(self, n, add_pypsa_cols=True):
         # Read in data from PyPSA
         bus_pypsa = n.buses
         sub_pypsa = pd.DataFrame()
-        bus2sub_pypsa = pd.DataFrame()
         gencost_cols = ["start_up_cost", "shut_down_cost", "marginal_cost"]
         gencost_pypsa = n.generators[gencost_cols]
         plant_pypsa = n.generators.drop(gencost_cols, axis=1)
@@ -207,37 +206,48 @@ def _read_network(self, n, add_pypsa_cols=True):
 
         # substations
         # only relevant if the PyPSA network was originally created from PSD
+        sub_cols = ["name", "interconnect_sub_id", "lat", "lon", "interconnect"]
+        sub_pypsa_cols = [
+            "y",
+            "x",
+        ]
         if "is_substation" in bus:
-            sub_cols = ["name", "interconnect_sub_id", "lat", "lon", "interconnect"]
             sub = bus[bus.is_substation][sub_cols]
             sub.index = sub[sub.index.str.startswith("sub")].index.str[3:]
-            sub_pypsa_cols = [
-                "name",
-                "interconnect_sub_id",
-                "y",
-                "x",
-                "interconnect",
-            ]
             sub_pypsa = bus_pypsa[bus_pypsa.is_substation][sub_pypsa_cols]
-            sub_pypsa.index = sub_pypsa[
-                sub_pypsa.index.str.startswith("sub")
-            ].index.str[3:]
+            sub_pypsa.index = sub.index
 
             bus = bus[~bus.is_substation]
             bus_pypsa = bus_pypsa[~bus_pypsa.is_substation]
 
             bus2sub = bus[["substation", "interconnect"]].copy()
             bus2sub["sub_id"] = pd.to_numeric(
-                bus2sub.pop("substation").str[3:], errors="ignore"
-            )
-            bus2sub_pypsa = bus_pypsa[["substation", "interconnect"]].copy()
-            bus2sub_pypsa["sub_id"] = pd.to_numeric(
-                bus2sub_pypsa.pop("substation").str[3:], errors="ignore"
+                bus2sub.pop("substation").str[3:], errors="coerce"
             )
         else:
-            warnings.warn("Substations could not be parsed.")
-            sub = pd.DataFrame()
-            bus2sub = pd.DataFrame()
+            # try to parse typical pypsa-eur(-sec) pattern for substations
+            sub_pattern = "[A-Z][A-Z]\d+\s\d+$"
+
+            sub = bus[bus.index.str.match(sub_pattern)].reindex(columns=sub_cols)
+            sub["interconnect"] = np.nan
+            sub["sub_id"] = sub.index
+            sub_pypsa = bus_pypsa[bus_pypsa.index.str.match(sub_pattern)][
+                sub_pypsa_cols
+            ]
+
+            sub_pattern = "([A-Z][A-Z]\d+\s\d+).*"
+            bus2sub = pd.DataFrame(
+                {
+                    "sub_id": bus.index.str.extract(sub_pattern)[0].values,
+                    "interconnect": np.nan,
+                },
+                index=bus.index,
+            )
+
+            if sub.empty and bus2sub.empty:
+                warnings.warn("Substations could not be parsed.")
+                sub = pd.DataFrame()
+                bus2sub = pd.DataFrame()
 
         # shunts
         # append PyPSA's shunts information to PSD's buses data frame on columns
@@ -332,7 +342,7 @@ def _read_network(self, n, add_pypsa_cols=True):
         values = [
             (bus, bus_pypsa, grid_const.col_name_bus),
             (sub, sub_pypsa, grid_const.col_name_sub),
-            (bus2sub, bus2sub_pypsa, grid_const.col_name_bus2sub),
+            (bus2sub, None, grid_const.col_name_bus2sub),
             (plant, plant_pypsa, grid_const.col_name_plant),
             (gencost, gencost_pypsa, grid_const.col_name_gencost),
             (branch, branch_pypsa, grid_const.col_name_branch),
@@ -367,7 +377,7 @@ def _read_network(self, n, add_pypsa_cols=True):
             df_psd = df_psd.reindex(const_location, axis="columns")
 
             # Add renamed PyPSA columns
-            if add_pypsa_cols:
+            if add_pypsa_cols and df_pypsa is not None:
                 df_pypsa = df_pypsa.add_prefix("pypsa_")
 
                 df_psd = pd.concat([df_psd, df_pypsa], axis=1)

From adf9309fc1af15498eae4d1580e15f51f5698888 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 28 Oct 2022 16:13:56 -0700
Subject: [PATCH 15/17] refactor: add inflow to column name of carriers with
 inflow profiles

---
 powersimdata/input/converter/pypsa_to_profiles.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/powersimdata/input/converter/pypsa_to_profiles.py b/powersimdata/input/converter/pypsa_to_profiles.py
index 09ee09fe6..b58ef8b35 100644
--- a/powersimdata/input/converter/pypsa_to_profiles.py
+++ b/powersimdata/input/converter/pypsa_to_profiles.py
@@ -36,12 +36,18 @@ def get_pypsa_gen_profile(network, profile2carrier):
     }
     profile = {}
     for p, c in profile2carrier.items():
+        c = [c] if isinstance(c, str) else list(c)
         profile[p] = pd.DataFrame()
         for component, ts in component2timeseries.items():
-            id_carrier = network.df(component).query("carrier==list(@c)").index
+            id_carrier = network.df(component).query("carrier==@c").index
             ts_carrier = get_switchable_as_dense(network, component, ts)[id_carrier]
             if not ts_carrier.empty:
-                norm = ts_carrier.max().replace(0, 1) if ts == "inflow" else 1
+                if ts == "inflow":
+                    has_inflow = ts_carrier.any().index[ts_carrier.any()]
+                    ts_carrier = ts_carrier[has_inflow].add_suffix(" inflow")
+                    norm = ts_carrier.max().replace(0, 1)
+                else:
+                    norm = 1
                 profile[p] = pd.concat([profile[p], ts_carrier / norm], axis=1)
 
         profile[p].rename_axis(index="UTC", columns=None, inplace=True)

From 04f2d4d76bfbeea62f8c0a6bea4dcc3ff669a8b9 Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Fri, 4 Nov 2022 21:20:54 +0100
Subject: [PATCH 16/17] feat: support hydro inflow functionality (#691)

---
 powersimdata/input/converter/pypsa_to_grid.py | 123 ++++++++++++------
 .../converter/tests/test_pypsa_to_grid.py     |  12 +-
 2 files changed, 94 insertions(+), 41 deletions(-)

diff --git a/powersimdata/input/converter/pypsa_to_grid.py b/powersimdata/input/converter/pypsa_to_grid.py
index a9f0eecac..f0973bade 100644
--- a/powersimdata/input/converter/pypsa_to_grid.py
+++ b/powersimdata/input/converter/pypsa_to_grid.py
@@ -31,27 +31,25 @@ def _invert_dict(d):
     return {v: k for k, v in d.items()}
 
 
-def _get_storage_storagedata(n, storage_type):
+def _get_storage_storagedata(df, storage_type):
     """Get storage data from PyPSA for data frame "StorageData" in PSD's
     storage dict.
 
-    :param pypsa.Network n: PyPSA network to read in.
+    :param pandas.DataFrame df: PyPSA component dataframe.
     :param str storage_type: key for PyPSA storage type.
     :return: (*pandas.DataFrame*) -- data frame with storage data.
     """
-    if storage_type == "storage_units":
+    storage_storagedata = _translate_df(df, "storage_storagedata")
 
-        storage_storagedata = _translate_df(n.storage_units, "storage_storagedata")
+    if storage_type == "storage_units":
 
-        e_nom = n.storage_units.eval("p_nom * max_hours")
-        state_of_charge_initial = n.storage_units["state_of_charge_initial"]
+        e_nom = df.eval("p_nom * max_hours")
+        state_of_charge_initial = df["state_of_charge_initial"]
 
     elif storage_type == "stores":
 
-        storage_storagedata = _translate_df(n.stores, "storage_storagedata")
-
-        e_nom = n.stores["e_nom"]
-        state_of_charge_initial = n.stores["e_initial"]
+        e_nom = df["e_nom"]
+        state_of_charge_initial = df["e_initial"]
 
         # Efficiencies of Store are captured in link/dcline
         storage_storagedata["OutEff"] = 1
@@ -83,27 +81,18 @@ def _get_storage_storagedata(n, storage_type):
     return storage_storagedata
 
 
-def _get_storage_gencost(n, storage_type):
+def _get_storage_gencost(df, storage_type):
     """Get storage data from PyPSA for data frame "gencost" in PSD's storage
     dict.
 
-    :param pypsa.Network n: PyPSA network to read in.
+    :param pandas.DataFrame df: PyPSA component dataframe.
     :param str storage_type: key for PyPSA storage type.
     :return: (*pandas.DataFrame*) -- data frame with storage data.
     """
-    if storage_type == "storage_units":
-        df_gencost = n.storage_units
-    elif storage_type == "stores":
-        df_gencost = n.stores
-    else:
-        warnings.warn(
-            "Inapplicable storage_type passed to function _get_storage_gencost."
-        )
-
     # There are "type" columns in gen and gencost with "type" column reserved
     # for gen dataframe, hence drop it here before renaming
-    df_gencost = df_gencost.drop(columns="type", errors="ignore")
-    storage_gencost = _translate_df(df_gencost, "storage_gencost")
+    df = df.drop(columns="type", errors="ignore")
+    storage_gencost = _translate_df(df, "storage_gencost")
     storage_gencost.assign(type=2, n=3, c0=0, c2=0)
     if "type" in storage_gencost:
         storage_gencost["type"] = pd.to_numeric(storage_gencost.type, errors="ignore")
@@ -111,26 +100,26 @@ def _get_storage_gencost(n, storage_type):
     return storage_gencost
 
 
-def _get_storage_gen(n, storage_type):
+def _get_storage_gen(df, storage_type):
     """Get storage data from PyPSA for data frame "gen" in PSD's storage dict.
 
-    :param pypsa.Network n: PyPSA network to read in.
+    :param pandas.DataFrame df: PyPSA component dataframe.
     :param str storage_type: key for PyPSA storage type.
     :return: (*pandas.DataFrame*) -- data frame with storage data.
     """
     if storage_type == "storage_units":
-        df_gen = n.storage_units
-        p_nom = n.storage_units["p_nom"]
+        pmax = df["p_nom"] * df["p_max_pu"]
+        pmin = df["p_nom"] * df["p_min_pu"]
     elif storage_type == "stores":
-        df_gen = n.stores
-        p_nom = np.inf
+        pmax = np.inf
+        pmin = -np.inf
     else:
         warnings.warn("Inapplicable storage_type passed to function _get_storage_gen.")
 
-    storage_gen = _translate_df(df_gen, "storage_gen")
-    storage_gen["Pmax"] = +p_nom
-    storage_gen["Pmin"] = -p_nom
-    storage_gen["ramp_30"] = p_nom
+    storage_gen = _translate_df(df, "storage_gen")
+    storage_gen["Pmax"] = pmax
+    storage_gen["Pmin"] = pmin
+    storage_gen["ramp_30"] = pmax
     storage_gen["Vg"] = 1
     storage_gen["mBase"] = 100
     storage_gen["status"] = 1
@@ -296,13 +285,67 @@ def _read_network(self, n, add_pypsa_cols=True):
         dcline["from_bus_id"] = pd.to_numeric(dcline.from_bus_id, errors="ignore")
         dcline["to_bus_id"] = pd.to_numeric(dcline.to_bus_id, errors="ignore")
 
-        # storage
-        storage_gen_storageunits = _get_storage_gen(n, "storage_units")
-        storage_gencost_storageunits = _get_storage_gencost(n, "storage_units")
-        storage_storagedata_storageunits = _get_storage_storagedata(n, "storage_units")
-        storage_gen_stores = _get_storage_gen(n, "stores")
-        storage_gencost_stores = _get_storage_gencost(n, "stores")
-        storage_storagedata_stores = _get_storage_storagedata(n, "stores")
+        # storage units
+        c = "storage_units"
+        storage_gen_storageunits = _get_storage_gen(n.storage_units, c)
+        storage_gencost_storageunits = _get_storage_gencost(n.storage_units, c)
+        storage_storagedata_storageunits = _get_storage_storagedata(n.storage_units, c)
+
+        inflow = n.get_switchable_as_dense("StorageUnit", "inflow")
+        has_inflow = inflow.any()
+        if has_inflow.any():
+            # add artificial buses
+            suffix = " inflow"
+
+            def add_suffix(s):
+                return str(s) + suffix
+
+            storage_gen_inflow = storage_gen_storageunits[has_inflow]
+            buses_old = storage_gen_inflow.bus_id.astype(str)
+            buses_new = storage_gen_inflow.index
+            bus_rename = dict(zip(buses_old, buses_new))
+            bus_inflow = bus.reindex(buses_old).rename(index=bus_rename)
+
+            # add discharging dcline (has same index as inflow storages)
+            dcline_inflow = pd.DataFrame(
+                {
+                    "from_bus_id": buses_new,
+                    "to_bus_id": buses_old,
+                    "Pmax": storage_gen_inflow.Pmax,
+                    "Pmin": storage_gen_inflow.Pmin,
+                }
+            )
+
+            # add inflow generator
+            gen_inflow = storage_gen_inflow.rename(index=add_suffix)
+            gen_inflow["Pmax"] = n.storage_units_t.inflow.max().rename(add_suffix)
+            gen_inflow["capital_cost"] = 0.0
+            gen_inflow["p_nom_extendable"] = False
+            gen_inflow["committable"] = False
+            gen_inflow["type"] = "inflow"
+            gen_inflow = gen_inflow.reindex(columns=plant.columns)
+            gencost_inflow = storage_gencost_storageunits[has_inflow].rename(
+                index=add_suffix
+            )
+            gencost_inflow = storage_gencost_storageunits.assign(
+                c0=0, c1=0, c2=0, type=2, startup=0, shutdown=0, n=3
+            )
+
+            # add everything to data
+            storage_gen_storageunits.loc[has_inflow, "bus_id"] = buses_new
+            storage_gen_storageunits.loc[
+                has_inflow, "Pmin"
+            ] = -np.inf  # don't limit charging from inflow
+            bus = pd.concat([bus, bus_inflow])
+            plant = pd.concat([plant, gen_inflow])
+            gencost = pd.concat([gencost, gencost_inflow])
+            dcline = pd.concat([dcline, dcline_inflow])
+
+        # stores
+        c = "stores"
+        storage_gen_stores = _get_storage_gen(n.stores, c)
+        storage_gencost_stores = _get_storage_gencost(n.stores, c)
+        storage_storagedata_stores = _get_storage_storagedata(n.stores, c)
         storage_genfuel = list(n.storage_units.carrier) + list(n.stores.carrier)
 
         # Pull operational properties into grid object
diff --git a/powersimdata/input/converter/tests/test_pypsa_to_grid.py b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
index c696c5a7c..16e052e65 100644
--- a/powersimdata/input/converter/tests/test_pypsa_to_grid.py
+++ b/powersimdata/input/converter/tests/test_pypsa_to_grid.py
@@ -28,8 +28,18 @@ def test_import_network_including_storages_from_pypsa_to_grid():
     n = pypsa.examples.storage_hvdc()
     grid = FromPyPSA(n)
 
+    inflow = n.get_switchable_as_dense("StorageUnit", "inflow")
+    has_inflow = inflow.any()
+
     assert not grid.bus.empty
-    assert len(n.buses) == len(grid.bus)
+    assert len(n.buses) + has_inflow.sum() == len(grid.bus)
+    assert len(n.generators) + has_inflow.sum() == len(grid.plant)
+    assert all(
+        [
+            "inflow" in i
+            for i in grid.plant.iloc[len(grid.plant) - has_inflow.sum() :].index
+        ]
+    )
     assert not grid.storage["gen"].empty
     assert not grid.storage["gencost"].empty
     assert not grid.storage["StorageData"].empty

From 3d17f0f5816cdf3f3ce45f0f4bc373df9d2990d5 Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Thu, 10 Nov 2022 22:11:31 +0100
Subject: [PATCH 17/17] fix: add geographical coordinates to branch and plant
 data frames and fix bus assignment/naming (#703)

---
 powersimdata/input/converter/pypsa_to_grid.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/powersimdata/input/converter/pypsa_to_grid.py b/powersimdata/input/converter/pypsa_to_grid.py
index f0973bade..1fd08995b 100644
--- a/powersimdata/input/converter/pypsa_to_grid.py
+++ b/powersimdata/input/converter/pypsa_to_grid.py
@@ -249,6 +249,8 @@ def _read_network(self, n, add_pypsa_cols=True):
         plant = _translate_df(df, "generator")
         plant["ramp_30"] = n.generators["ramp_limit_up"].fillna(0)
         plant["Pmin"] *= plant["Pmax"]  # from relative to absolute value
+        plant["lat"] = plant.bus_id.map(bus.lat)
+        plant["lon"] = plant.bus_id.map(bus.lon)
         plant["bus_id"] = pd.to_numeric(plant.bus_id, errors="ignore")
 
         # generation costs
@@ -276,6 +278,10 @@ def _read_network(self, n, add_pypsa_cols=True):
         # BE model assumes a 100 MVA base, pypsa "assumes" a 1 MVA base
         branch["x"] *= 100
         branch["r"] *= 100
+        branch["from_lat"] = branch.from_bus_id.map(bus.lat)
+        branch["from_lon"] = branch.from_bus_id.map(bus.lon)
+        branch["to_lat"] = branch.to_bus_id.map(bus.lat)
+        branch["to_lon"] = branch.to_bus_id.map(bus.lon)
         branch["from_bus_id"] = pd.to_numeric(branch.from_bus_id, errors="ignore")
         branch["to_bus_id"] = pd.to_numeric(branch.to_bus_id, errors="ignore")
 
@@ -305,6 +311,7 @@ def add_suffix(s):
             buses_new = storage_gen_inflow.index
             bus_rename = dict(zip(buses_old, buses_new))
             bus_inflow = bus.reindex(buses_old).rename(index=bus_rename)
+            bus2sub_inflow = bus2sub.reindex(buses_old).rename(index=bus_rename)
 
             # add discharging dcline (has same index as inflow storages)
             dcline_inflow = pd.DataFrame(
@@ -318,11 +325,14 @@ def add_suffix(s):
 
             # add inflow generator
             gen_inflow = storage_gen_inflow.rename(index=add_suffix)
+            gen_inflow["bus_id"] = buses_new
             gen_inflow["Pmax"] = n.storage_units_t.inflow.max().rename(add_suffix)
             gen_inflow["capital_cost"] = 0.0
             gen_inflow["p_nom_extendable"] = False
             gen_inflow["committable"] = False
             gen_inflow["type"] = "inflow"
+            gen_inflow["lat"] = gen_inflow.bus_id.map(bus.lat)
+            gen_inflow["lon"] = gen_inflow.bus_id.map(bus.lon)
             gen_inflow = gen_inflow.reindex(columns=plant.columns)
             gencost_inflow = storage_gencost_storageunits[has_inflow].rename(
                 index=add_suffix
@@ -337,6 +347,7 @@ def add_suffix(s):
                 has_inflow, "Pmin"
             ] = -np.inf  # don't limit charging from inflow
             bus = pd.concat([bus, bus_inflow])
+            bus2sub = pd.concat([bus2sub, bus2sub_inflow])
             plant = pd.concat([plant, gen_inflow])
             gencost = pd.concat([gencost, gencost_inflow])
             dcline = pd.concat([dcline, dcline_inflow])