Update the dat -> csv file transition code to use 2 columns (name,val…

…ue) rather than having a wide table with 2 rows.

switch_model/balancing/operating_reserves/spinning_reserves.py

@@ -87,8 +87,11 @@
 
 """
 import os
+
 from pyomo.environ import *
 
+from switch_model.utilities import load_key_value_inputfile
+
 dependencies = (
     'switch_model.timescales',
     'switch_model.balancing.load_zones',
@@ -521,17 +524,18 @@ def load_inputs(m, switch_data, inputs_dir):
         GENERATION_PROJECTS, ... gen_can_provide_spinning_reserves
 
     spinning_reserve_params.csv may override the default value of
-    contingency_safety_factor. Note that this only contains one
-    header row and one data row.
+    contingency_safety_factor. This should have column headers "name,value"
+    with one parameter per row.
+
     """
     switch_data.load_aug(
         filename=os.path.join(inputs_dir, 'generation_projects_info.csv'),
         auto_select=True,
         optional_params=['gen_can_provide_spinning_reserves'],
         param=(m.gen_can_provide_spinning_reserves)
     )
-    switch_data.load_aug(
-        filename=os.path.join(inputs_dir, 'spinning_reserve_params.csv'),
-        optional=True, auto_select=True,
-        param=(m.contingency_safety_factor,)
+    load_key_value_inputfile(
+        switch_data,
+        optional=True,
+        filename=os.path.join(inputs_dir, 'spinning_reserve_params.csv')
     )

switch_model/balancing/operating_reserves/spinning_reserves_advanced.py

@@ -4,11 +4,13 @@
 This is an advanced version of the basic spinning_reserves reserves module, and
 can be used in place of it (not in addition to).
 """
-import os
 from collections import defaultdict
+import os
+
 from pyomo.environ import *
-from switch_model.utilities import iteritems
 
+from switch_model.utilities import iteritems
+from switch_model.utilities import load_key_value_inputfile
 
 dependencies = (
     'switch_model.timescales',
@@ -609,8 +611,8 @@ def load_inputs(m, switch_data, inputs_dir):
         GENERATION_PROJECTS, RESERVE_TYPES, [gen_reserve_type_max_share]
 
     spinning_reserve_params.csv may override the default value of
-    contingency_safety_factor. Note that this only contains one header row
-    and one data row.
+    contingency_safety_factor. This should have column headers "name,value"
+    with one parameter per row.
     """
     path=os.path.join(inputs_dir, 'generation_projects_reserve_capability.csv')
     switch_data.load_aug(
@@ -627,8 +629,8 @@ def load_inputs(m, switch_data, inputs_dir):
         switch_data.data()['GEN_SPINNING_RESERVE_TYPES'][None] = \
             [(g, "spinning") for g in gen_projects]
 
-    switch_data.load_aug(
-        filename=os.path.join(inputs_dir, 'spinning_reserve_params.csv'),
-        optional=True, auto_select=True,
-        param=(m.contingency_safety_factor,)
+    load_key_value_inputfile(
+        switch_data,
+        optional=True,
+        filename=os.path.join(inputs_dir, 'spinning_reserve_params.csv')
     )

switch_model/balancing/unserved_load.py

@@ -9,8 +9,11 @@
 """
 
 import os
+
 from pyomo.environ import *
 
+from switch_model.utilities import load_key_value_inputfile
+
 dependencies = 'switch_model.timescales',\
     'switch_model.balancing.load_areas', 'switch_model.financials'
 
@@ -48,17 +51,16 @@ def define_components(mod):
 def load_inputs(mod, switch_data, inputs_dir):
     """
     The cost penalty of unserved load in units of $/MWh is the only parameter
-    that can be inputted. The following file is not mandatory, because the
-    parameter defaults to a value of 500 $/MWh. This file contains one header
-    row and one data row.
-
-    optional input files:
-        lost_load_cost.csv
-            unserved_load_penalty
+    that can be inputted. lost_load_cost.csv is optional, because 
+    unserved_load_penalty defaults to a value of 500 $/MWh. 
+    
+    lost_load_cost.csv: 
+        headers = name,value
+        allowed parameters: unserved_load_penalty
 
     """
-    switch_data.load_aug(
-        filename=os.path.join(inputs_dir, 'lost_load_cost.csv'),
-        optional=True, auto_select=True,
-        param=(mod.unserved_load_penalty,)
+    load_key_value_inputfile(
+        switch_data,
+        optional=True,
+        filename=os.path.join(inputs_dir, 'lost_load_cost.csv')
     )

switch_model/financials.py

@@ -7,9 +7,12 @@
 """
 from __future__ import print_function
 from __future__ import division
-from pyomo.environ import *
 import os
+
 import pandas as pd
+from pyomo.environ import *
+
+from switch_model.utilities import load_key_value_inputfile
 
 dependencies = 'switch_model.timescales'
 
@@ -309,14 +312,12 @@ def load_inputs(mod, switch_data, inputs_dir):
     """
     Import base financial data from a .csv file. The inputs_dir should
     contain the file financials.csv that gives parameter values for
-    base_financial_year, interest_rate and optionally discount_rate.
-    The names of parameters go on the first row and the values go on
-    the second.
+    base_financial_year, interest_rate and optionally discount_rate. 
+    These should be specified one-per row, with column headers "name,value"
     """
-    switch_data.load_aug(
-        filename=os.path.join(inputs_dir, 'financials.csv'),
-        optional=False, auto_select=True,
-        param=(mod.base_financial_year, mod.interest_rate, mod.discount_rate)
+    load_key_value_inputfile(
+        switch_data,
+        filename=os.path.join(inputs_dir, 'financials.csv')
     )
 
 def post_solve(instance, outdir):

switch_model/generators/extensions/hydro_system.py

@@ -30,8 +30,11 @@
 from __future__ import division
 
 import os
+
 from pyomo.environ import *
 
+from switch_model.utilities import load_key_value_inputfile
+
 dependencies = 'switch_model.timescales', 'switch_model.balancing.load_zones',\
     'switch_model.financials', 'switch_model.energy_sources.properties.properties', \
     'switch_model.generators.core.build', 'switch_model.generators.core.dispatch'
@@ -460,6 +463,9 @@ def load_inputs(mod, switch_data, inputs_dir):
     hydro is treated like any other variable renewable resource, and
     expects data in variable_capacity_factors.csv.
 
+    The optional file spillage_penalty.csv may override the default value of
+    spillage_penalty. This should have column headers "name,value" with one
+    parameter per row.
     """
 
     switch_data.load_aug(
@@ -508,8 +514,8 @@ def load_inputs(mod, switch_data, inputs_dir):
         auto_select=True,
         index=mod.HYDRO_GENS,
         param=(mod.hydro_efficiency, mod.hydraulic_location))
-    switch_data.load_aug(
-        filename=os.path.join(inputs_dir, 'spillage_penalty.csv'),
-        optional=True, auto_select=True,
-        param=(mod.spillage_penalty,)
+    load_key_value_inputfile(
+        switch_data,
+        optional=True,
+        filename=os.path.join(inputs_dir, 'spillage_penalty.csv')
     )

switch_model/hawaii/hydrogen.py

@@ -1,7 +1,10 @@
 from __future__ import division
 import os
+
 from pyomo.environ import *
+
 from switch_model.financials import capital_recovery_factor as crf
+from switch_model.utilities import load_key_value_inputfile
 
 def define_arguments(argparser):
     argparser.add_argument('--hydrogen-reserve-types', nargs='+', default=['spinning'],
@@ -264,27 +267,8 @@ def load_inputs(m, switch_data, inputs_dir):
     TODO: change this to allow multiple storage technologies.
     """
     if not m.options.no_hydrogen:
-        switch_data.load_aug(
-            filename=os.path.join(inputs_dir, 'hydrogen.csv'),
-            optional=False, auto_select=True,
-            param=(
-                m.hydrogen_electrolyzer_capital_cost_per_mw,
-                m.hydrogen_electrolyzer_fixed_cost_per_mw_year,
-                m.hydrogen_electrolyzer_kg_per_mwh,
-                m.hydrogen_electrolyzer_life_years,
-                m.hydrogen_electrolyzer_variable_cost_per_kg,
-                m.hydrogen_fuel_cell_capital_cost_per_mw,
-                m.hydrogen_fuel_cell_fixed_cost_per_mw_year,
-                m.hydrogen_fuel_cell_life_years,
-                m.hydrogen_fuel_cell_mwh_per_kg,
-                m.hydrogen_fuel_cell_variable_cost_per_mwh,
-                m.hydrogen_liquifier_capital_cost_per_kg_per_hour,
-                m.hydrogen_liquifier_fixed_cost_per_kg_hour_year,
-                m.hydrogen_liquifier_life_years,
-                m.hydrogen_liquifier_mwh_per_kg,
-                m.hydrogen_liquifier_variable_cost_per_kg,
-                m.liquid_hydrogen_tank_capital_cost_per_kg,
-                m.liquid_hydrogen_tank_life_years,
-                m.liquid_hydrogen_tank_minimum_size_kg,
-            )
+        load_key_value_inputfile(
+            switch_data,
+            optional=True,
+            filename=os.path.join(inputs_dir, 'hydrogen.csv')
         )

switch_model/hawaii/scenario_data.py

@@ -27,8 +27,12 @@
 
 import time, sys, collections, os
 from textwrap import dedent
+
+import pandas
+
 from switch_model import __version__ as switch_version
 from switch_model.utilities import iteritems
+from switch_model.utilities import load_key_value_inputfile
 
 # NOTE: instead of using the python csv writer, this directly writes tables to
 # file in a customized, pyomo-friendly .csv format. This uses commas between columns
@@ -951,28 +955,25 @@ def db_cursor():
 
 def write_simple_csv(output_file, args_to_write, arguments):
     """ write a simple .csv file with the arguments specified in args_to_write,
-    drawn from the arguments dictionary. This includes one row with all the
-    parameter names and a second row with their values.
+    drawn from the arguments dictionary. This has two columns (name & value),
+    with one row per parameter.
     (previously write_dat_file())"""
 
     start=time.time()
 
-    # collect data for the two rows (if any)
-    headers = []
-    values = []
+    # collect data for the two columns (if any)
+    headers = ["name", "value"]
+    dat = []
     for name in args_to_write:
         if name in arguments:
-            headers.append(name)
-            values.append(str(arguments[name]))
+            dat.append((str(name), str(arguments[name])))
 
-    if headers:
+    if dat:
         output_file = make_file_path(output_file, arguments)
         print("Writing {file} ...".format(file=output_file), end=' ')
         sys.stdout.flush()  # display the part line to the user
-
-        with open(output_file, 'w') as f:
-            f.write(','.join(headers) + '\n')
-            f.write(','.join(values) + '\n')
+        df = pandas.DataFrame(dat, columns=headers)
+        df.to_csv(output_file)
 
         print("time taken: {dur:.2f}s".format(dur=time.time()-start))
 

switch_model/transmission/transport/build.py

@@ -6,9 +6,12 @@
 """
 
 import os
+
+import pandas as pd
 from pyomo.environ import *
+
 from switch_model.financials import capital_recovery_factor as crf
-import pandas as pd
+from switch_model.utilities import load_key_value_inputfile
 
 dependencies = 'switch_model.timescales', 'switch_model.balancing.load_zones',\
     'switch_model.financials'
@@ -276,29 +279,26 @@ def init_trans_d_line(m, zone_from, zone_to):
 
 def load_inputs(mod, switch_data, inputs_dir):
     """
-
     Import data related to transmission builds. The following files are
-    expected in the input directory:
+    expected in the input directory. Optional files and columns are marked
+    with *. If optional columns are missing or if cells contain a dot (.),
+    those parameters will be set to default values as described in
+    documentation.
 
     transmission_lines.csv
         TRANSMISSION_LINE, trans_lz1, trans_lz2, trans_length_km,
-        trans_efficiency, existing_trans_cap, trans_dbid,
-        trans_derating_factor, trans_terrain_multiplier,
-        trans_new_build_allowed
-    The last 4 columns of transmission_lines.csv are optional. If the
-    columns are missing or if cells contain a dot (.), those parameters
-    will be set to default values as described in documentation.
-
-    Note that in the next file, parameter names are written on the first
-    row (as usual), and the single value for each parameter is written in
-    the second row. The distribution_loss_rate parameter is read by the
-    local_td module (if used).
-
-    trans_params.csv
+        trans_efficiency, existing_trans_cap, trans_dbid*,
+        trans_derating_factor*, trans_terrain_multiplier*,
+        trans_new_build_allowed*
+
+    trans_params.csv has two columns: name,value with one parameter per row.
+    The distribution_loss_rate parameter is read by the local_td module (if
+    used).
+
+    trans_params.csv*
         trans_capital_cost_per_mw_km, trans_lifetime_yrs,
         trans_fixed_om_fraction, distribution_loss_rate
     """
-
     # TODO: send issue / pull request to Pyomo to allow .csv files with
     # no rows after header (fix bugs in pyomo.core.plugins.data.text)
     switch_data.load_aug(
@@ -321,13 +321,10 @@ def load_inputs(mod, switch_data, inputs_dir):
             mod.trans_terrain_multiplier, mod.trans_new_build_allowed
         )
     )
-    switch_data.load_aug(
-        filename=os.path.join(inputs_dir, 'trans_params.csv'),
-        optional=True, auto_select=True,
-        param=(
-            mod.trans_capital_cost_per_mw_km, mod.trans_lifetime_yrs,
-            mod.trans_fixed_om_fraction, mod.distribution_loss_rate
-        )
+    load_key_value_inputfile(
+        switch_data,
+        optional=True,
+        filename=os.path.join(inputs_dir, 'trans_params.csv')
     )