From 88699a917e1e6b40c193fceda479cf830949cba2 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Mon, 3 May 2021 16:33:45 -0700
Subject: [PATCH 01/11] feat: modular config pattern

---
 powersimdata/data_access/context.py  |  2 +-
 powersimdata/scenario/execute.py     |  2 +-
 powersimdata/utility/config.py       | 54 ++++++++++++++++++++++++++++
 powersimdata/utility/server_setup.py | 34 +++++++-----------
 4 files changed, 68 insertions(+), 24 deletions(-)
 create mode 100644 powersimdata/utility/config.py

diff --git a/powersimdata/data_access/context.py b/powersimdata/data_access/context.py
index e14dcfb67..d84abbda9 100644
--- a/powersimdata/data_access/context.py
+++ b/powersimdata/data_access/context.py
@@ -1,6 +1,6 @@
 from powersimdata.data_access.data_access import LocalDataAccess, SSHDataAccess
 from powersimdata.utility import server_setup
-from powersimdata.utility.server_setup import DeploymentMode, get_deployment_mode
+from powersimdata.utility.config import DeploymentMode, get_deployment_mode
 
 
 class Context:
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 9a92e30bc..fdb86ce54 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -11,7 +11,7 @@
 from powersimdata.input.transform_profile import TransformProfile
 from powersimdata.scenario.state import State
 from powersimdata.utility import server_setup
-from powersimdata.utility.server_setup import DeploymentMode, get_deployment_mode
+from powersimdata.utility.config import DeploymentMode, get_deployment_mode
 
 
 class Execute(State):
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
new file mode 100644
index 000000000..498c3df54
--- /dev/null
+++ b/powersimdata/utility/config.py
@@ -0,0 +1,54 @@
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Config:
+    DATA_ROOT_DIR = "/mnt/bes/pcm"
+    EXECUTE_DIR = "tmp"
+    INPUT_DIR = ("data", "input")
+    OUTPUT_DIR = ("data", "output")
+    LOCAL_DIR = os.path.join(Path.home(), "ScenarioData", "")
+    MODEL_DIR = "/home/bes/pcm"
+
+
+@dataclass(frozen=True)
+class ServerConfig(Config):
+    SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
+    SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
+    BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2"
+
+
+@dataclass(frozen=True)
+class ContainerConfig(Config):
+    SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "reisejl")
+
+
+@dataclass(frozen=True)
+class LocalConfig(Config):
+    DATA_ROOT_DIR = Config.LOCAL_DIR
+
+
+class DeploymentMode:
+    Server = "SERVER"
+    Container = "CONTAINER"
+    Local = "LOCAL"
+
+    CONFIG_MAP = {Server: ServerConfig, Container: ContainerConfig, Local: LocalConfig}
+
+
+def get_deployment_mode():
+    # TODO: consider auto detection
+    mode = os.getenv("DEPLOYMENT_MODE")
+    if mode is None:
+        return DeploymentMode.Server
+    if mode == "1" or mode.lower() == "container":
+        return DeploymentMode.Container
+    if mode == "2" or mode.lower() == "local":
+        return DeploymentMode.Local
+
+
+def get_config():
+    mode = get_deployment_mode()
+    return DeploymentMode.CONFIG_MAP[mode]()
diff --git a/powersimdata/utility/server_setup.py b/powersimdata/utility/server_setup.py
index e65a74e45..55d7f84ad 100644
--- a/powersimdata/utility/server_setup.py
+++ b/powersimdata/utility/server_setup.py
@@ -1,27 +1,17 @@
 import os
-from pathlib import Path
 
-SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
-SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
-BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2"
-DATA_ROOT_DIR = "/mnt/bes/pcm"
-EXECUTE_DIR = "tmp"
-INPUT_DIR = ("data", "input")
-OUTPUT_DIR = ("data", "output")
-LOCAL_DIR = os.path.join(Path.home(), "ScenarioData", "")
-MODEL_DIR = "/home/bes/pcm"
-
-
-class DeploymentMode:
-    Server = "SERVER"
-    Container = "CONTAINER"
-
-
-def get_deployment_mode():
-    mode = os.getenv("DEPLOYMENT_MODE")
-    if mode is None:
-        return DeploymentMode.Server
-    return DeploymentMode.Container
+from powersimdata.utility.config import get_config
+
+config = get_config()
+SERVER_ADDRESS = config.SERVER_ADDRESS
+SERVER_SSH_PORT = config.SERVER_SSH_PORT
+BACKUP_DATA_ROOT_DIR = config.BACKUP_DATA_ROOT_DIR
+DATA_ROOT_DIR = config.DATA_ROOT_DIR
+EXECUTE_DIR = config.EXECUTE_DIR
+INPUT_DIR = config.INPUT_DIR
+OUTPUT_DIR = config.OUTPUT_DIR
+LOCAL_DIR = config.LOCAL_DIR
+MODEL_DIR = config.MODEL_DIR
 
 
 def get_server_user():

From e3a1fac5f05d7f82f301316fdd95010592ae2e30 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Tue, 4 May 2021 12:30:36 -0700
Subject: [PATCH 02/11] refactor: split launch_simulation into subclasses

---
 powersimdata/data_access/context.py  |  14 +++
 powersimdata/data_access/launcher.py | 139 +++++++++++++++++++++++++++
 powersimdata/scenario/execute.py     | 128 ++----------------------
 3 files changed, 163 insertions(+), 118 deletions(-)
 create mode 100644 powersimdata/data_access/launcher.py

diff --git a/powersimdata/data_access/context.py b/powersimdata/data_access/context.py
index d84abbda9..afb7caa38 100644
--- a/powersimdata/data_access/context.py
+++ b/powersimdata/data_access/context.py
@@ -1,4 +1,5 @@
 from powersimdata.data_access.data_access import LocalDataAccess, SSHDataAccess
+from powersimdata.data_access.launcher import HttpLauncher, NativeLauncher, SSHLauncher
 from powersimdata.utility import server_setup
 from powersimdata.utility.config import DeploymentMode, get_deployment_mode
 
@@ -23,3 +24,16 @@ def get_data_access(data_loc=None):
         if mode == DeploymentMode.Server:
             return SSHDataAccess(root)
         return LocalDataAccess(root)
+
+    @staticmethod
+    def get_launcher(scenario):
+        """Return instance for interaction with simulation engine
+
+        :param powersimdata.Scenario scenario: a scenario object
+        """
+        mode = get_deployment_mode()
+        if mode == DeploymentMode.Server:
+            return SSHLauncher(scenario)
+        elif mode == DeploymentMode.Container:
+            return HttpLauncher(scenario)
+        return NativeLauncher(scenario)
diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
new file mode 100644
index 000000000..bc53acc31
--- /dev/null
+++ b/powersimdata/data_access/launcher.py
@@ -0,0 +1,139 @@
+import posixpath
+
+import requests
+
+from powersimdata.utility import server_setup
+
+
+def _check_threads(threads):
+    """Validate threads argument
+
+    :param int threads: the number of threads to be used
+    :raises TypeError: if threads is not an int
+    :raises ValueError: if threads is not a positive value
+    """
+    if threads:
+        if not isinstance(threads, int):
+            raise TypeError("threads must be an int")
+        if threads < 1:
+            raise ValueError("threads must be a positive value")
+
+
+def _check_solver(solver):
+    """Validate solver argument
+
+    :param str solver: the solver used for the optimization
+    :raises ValueError: if invalid solver provided
+    """
+    solvers = ("gurobi", "glpk")
+    if solver is not None and solver.lower() not in solvers:
+        raise ValueError(f"Invalid solver: options are {solvers}")
+
+
+# TODO - check_progress (just print a message for SSHLauncher)
+# TODO - extract_data
+class Launcher:
+    def __init__(self, scenario):
+        self.scenario = scenario
+
+    def _launch(self, threads=None, solver=None, extract_data=True):
+        raise NotImplementedError
+
+    def launch_simulation(self, threads=None, solver=None, extract_data=True):
+        _check_threads(threads)
+        _check_solver(solver)
+        self._launch(threads, solver, extract_data)
+
+
+class SSHLauncher(Launcher):
+    def _run_script(self, script, extra_args=None):
+        """Returns running process
+
+        :param str script: script to be used.
+        :param list extra_args: list of strings to be passed after scenario id.
+        :return: (*subprocess.Popen*) -- process used to run script
+        """
+        if extra_args is None:
+            extra_args = []
+
+        engine = self.scenario._scenario_info["engine"]
+        path_to_package = posixpath.join(server_setup.MODEL_DIR, engine)
+        folder = "pyreise" if engine == "REISE" else "pyreisejl"
+
+        path_to_script = posixpath.join(path_to_package, folder, "utility", script)
+        cmd_pythonpath = [f'export PYTHONPATH="{path_to_package}:$PYTHONPATH";']
+        cmd_pythoncall = [
+            "nohup",
+            "python3",
+            "-u",
+            path_to_script,
+            self.scenario.scenario_id,
+        ]
+        cmd_io_redirect = ["</dev/null >/dev/null 2>&1 &"]
+        cmd = cmd_pythonpath + cmd_pythoncall + extra_args + cmd_io_redirect
+        process = self.scenario._data_access.execute_command_async(cmd)
+        print("PID: %s" % process.pid)
+        return process
+
+    def _launch(self, threads=None, solver=None, extract_data=True):
+        """Launch simulation on server, via ssh.
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: whether the results of the simulation engine should
+            automatically extracted after the simulation has run. This defaults to True.
+        :raises TypeError: if extract_data is not a boolean
+        :return: (*subprocess.Popen*) -- new process used to launch simulation.
+        """
+        extra_args = []
+
+        if threads:
+            # Use the -t flag as defined in call.py in REISE.jl
+            extra_args.append("--threads " + str(threads))
+
+        if solver:
+            extra_args.append("--solver " + solver)
+
+        if not isinstance(extract_data, bool):
+            raise TypeError("extract_data must be a boolean: 'True' or 'False'")
+        if extract_data:
+            extra_args.append("--extract-data")
+
+        return self._run_script("call.py", extra_args=extra_args)
+
+
+class HttpLauncher(Launcher):
+    def _launch(self, threads=None, solver=None, extract_data=True):
+        """Launches simulation in container via http call
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: always True
+        :return: (*requests.Response*) -- the http response object
+        """
+        scenario_id = self.scenario.scenario_id
+        url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}"
+        resp = requests.post(url, params={"threads": threads, "solver": solver})
+        if resp.status_code != 200:
+            print(
+                f"Failed to launch simulation: status={resp.status_code}. See response for details"
+            )
+        return resp
+
+
+class NativeLauncher(Launcher):
+    def _launch(self, threads=None, solver=None, extract_data=True):
+        """Launches simulation by importing from REISE.jl
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: always True
+        :return: (*dict*) -- json response
+        """
+        pass
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index fdb86ce54..7248786d0 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -4,6 +4,7 @@
 
 import requests
 
+from powersimdata.data_access.context import Context
 from powersimdata.input.case_mat import export_case_mat
 from powersimdata.input.grid import Grid
 from powersimdata.input.input_data import InputData
@@ -31,6 +32,7 @@ class Execute(State):
         "prepare_simulation_input",
         "print_scenario_info",
         "print_scenario_status",
+        "scenario_id",
     }
 
     def __init__(self, scenario):
@@ -47,8 +49,10 @@ def __init__(self, scenario):
         print("--> Status\n%s" % self._scenario_status)
 
         self._set_ct_and_grid()
+        self._launcher = Context.get_launcher(scenario)
 
-    def _scenario_id(self):
+    @property
+    def scenario_id(self):
         return self._scenario_info["id"]
 
     def _set_ct_and_grid(self):
@@ -81,50 +85,11 @@ def get_grid(self):
 
     def _update_scenario_status(self):
         """Updates scenario status."""
-        self._scenario_status = self._execute_list_manager.get_status(
-            self._scenario_id()
-        )
+        self._scenario_status = self._execute_list_manager.get_status(self.scenario_id)
 
     def _update_scenario_info(self):
         """Updates scenario information."""
-        self._scenario_info = self._scenario_list_manager.get_scenario(
-            self._scenario_id()
-        )
-
-    def _run_script(self, script, extra_args=None):
-        """Returns running process
-
-        :param str script: script to be used.
-        :param list extra_args: list of strings to be passed after scenario id.
-        :return: (*subprocess.Popen*) -- process used to run script
-        """
-
-        if not extra_args:
-            extra_args = []
-
-        path_to_package = posixpath.join(
-            server_setup.MODEL_DIR, self._scenario_info["engine"]
-        )
-
-        if self._scenario_info["engine"] == "REISE":
-            folder = "pyreise"
-        else:
-            folder = "pyreisejl"
-
-        path_to_script = posixpath.join(path_to_package, folder, "utility", script)
-        cmd_pythonpath = [f'export PYTHONPATH="{path_to_package}:$PYTHONPATH";']
-        cmd_pythoncall = [
-            "nohup",
-            "python3",
-            "-u",
-            path_to_script,
-            self._scenario_info["id"],
-        ]
-        cmd_io_redirect = ["</dev/null >/dev/null 2>&1 &"]
-        cmd = cmd_pythonpath + cmd_pythoncall + extra_args + cmd_io_redirect
-        process = self._data_access.execute_command_async(cmd)
-        print("PID: %s" % process.pid)
-        return process
+        self._scenario_info = self._scenario_list_manager.get_scenario(self.scenario_id)
 
     def print_scenario_info(self):
         """Prints scenario information."""
@@ -167,7 +132,7 @@ def prepare_simulation_input(self, profiles_as=None):
 
             si.prepare_mpc_file()
 
-            self._execute_list_manager.set_status(self._scenario_id(), "prepared")
+            self._execute_list_manager.set_status(self.scenario_id, "prepared")
         else:
             print("---------------------------")
             print("SCENARIO CANNOT BE PREPARED")
@@ -187,75 +152,6 @@ def _check_if_ready(self):
                 f"Status must be one of {valid_status}, but got status={self._scenario_status}"
             )
 
-    def _launch_on_server(self, threads=None, solver=None, extract_data=True):
-        """Launch simulation on server, via ssh.
-
-        :param int/None threads: the number of threads to be used. This defaults to None,
-            where None means auto.
-        :param str solver: the solver used for optimization. This defaults to
-            None, which translates to gurobi
-        :param bool extract_data: whether the results of the simulation engine should
-            automatically extracted after the simulation has run. This defaults to True.
-        :raises TypeError: if extract_data is not a boolean
-        :return: (*subprocess.Popen*) -- new process used to launch simulation.
-        """
-        extra_args = []
-
-        if threads:
-            # Use the -t flag as defined in call.py in REISE.jl
-            extra_args.append("--threads " + str(threads))
-
-        if solver:
-            extra_args.append("--solver " + solver)
-
-        if not isinstance(extract_data, bool):
-            raise TypeError("extract_data must be a boolean: 'True' or 'False'")
-        if extract_data:
-            extra_args.append("--extract-data")
-
-        return self._run_script("call.py", extra_args=extra_args)
-
-    def _launch_in_container(self, threads, solver):
-        """Launches simulation in container via http call
-
-        :param int/None threads: the number of threads to be used. This defaults to None,
-            where None means auto.
-        :param str solver: the solver used for optimization. This defaults to
-            None, which translates to gurobi
-        :return: (*requests.Response*) -- the http response object
-        """
-        scenario_id = self._scenario_id()
-        url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}"
-        resp = requests.post(url, params={"threads": threads, "solver": solver})
-        if resp.status_code != 200:
-            print(
-                f"Failed to launch simulation: status={resp.status_code}. See response for details"
-            )
-        return resp
-
-    def _check_threads(self, threads):
-        """Validate threads argument
-
-        :param int threads: the number of threads to be used
-        :raises TypeError: if threads is not an int
-        :raises ValueError: if threads is not a positive value
-        """
-        if threads:
-            if not isinstance(threads, int):
-                raise TypeError("threads must be an int")
-            if threads < 1:
-                raise ValueError("threads must be a positive value")
-
-    def _check_solver(self, solver):
-        """Validate solver argument
-
-        :param str solver: the solver used for the optimization
-        :raises ValueError: if invalid solver provided
-        """
-        solvers = ("gurobi", "glpk")
-        if solver is not None and solver.lower() not in solvers:
-            raise ValueError(f"Invalid solver: options are {solvers}")
-
     def launch_simulation(self, threads=None, extract_data=True, solver=None):
         """Launches simulation on target environment (server or container)
 
@@ -269,14 +165,10 @@ def launch_simulation(self, threads=None, extract_data=True, solver=None):
             process (if using ssh to server) or http response (if run in container)
         """
         self._check_if_ready()
-        self._check_threads(threads)
-        self._check_solver(solver)
 
         mode = get_deployment_mode()
         print(f"--> Launching simulation on {mode.lower()}")
-        if mode == DeploymentMode.Server:
-            return self._launch_on_server(threads, solver, extract_data)
-        return self._launch_in_container(threads, solver)
+        self._launcher.launch_simulation(threads, extract_data, solver)
 
     def check_progress(self):
         """Get the lastest information from the server container
@@ -287,7 +179,7 @@ def check_progress(self):
         if mode != DeploymentMode.Container:
             raise NotImplementedError("Operation only supported for container mode")
 
-        scenario_id = self._scenario_id()
+        scenario_id = self.scenario_id
         url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}"
         resp = requests.get(url)
         return resp.json()

From c5bf5e63f7ff74cf9c6a52072dbeb9315a7b629d Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Tue, 4 May 2021 13:57:15 -0700
Subject: [PATCH 03/11] refactor: split up other launcher methods

---
 powersimdata/data_access/launcher.py | 40 ++++++++++++++++++++++++----
 powersimdata/scenario/execute.py     | 25 ++++-------------
 2 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index bc53acc31..a6e77a137 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -30,8 +30,6 @@ def _check_solver(solver):
         raise ValueError(f"Invalid solver: options are {solvers}")
 
 
-# TODO - check_progress (just print a message for SSHLauncher)
-# TODO - extract_data
 class Launcher:
     def __init__(self, scenario):
         self.scenario = scenario
@@ -39,6 +37,10 @@ def __init__(self, scenario):
     def _launch(self, threads=None, solver=None, extract_data=True):
         raise NotImplementedError
 
+    def extract_simulation_output(self):
+        """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} on server."""
+        pass
+
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
         _check_threads(threads)
         _check_solver(solver)
@@ -88,12 +90,11 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :return: (*subprocess.Popen*) -- new process used to launch simulation.
         """
         extra_args = []
-
-        if threads:
+        if threads is not None:
             # Use the -t flag as defined in call.py in REISE.jl
             extra_args.append("--threads " + str(threads))
 
-        if solver:
+        if solver is not None:
             extra_args.append("--solver " + solver)
 
         if not isinstance(extract_data, bool):
@@ -103,6 +104,18 @@ def _launch(self, threads=None, solver=None, extract_data=True):
 
         return self._run_script("call.py", extra_args=extra_args)
 
+    def extract_simulation_output(self):
+        """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} on server.
+
+        :return: (*subprocess.Popen*) -- new process used to extract output
+            data.
+        """
+        print("--> Extracting output data on server")
+        return self._run_script("extract_data.py")
+
+    def check_progress(self):
+        print("Information is available on the server.")
+
 
 class HttpLauncher(Launcher):
     def _launch(self, threads=None, solver=None, extract_data=True):
@@ -124,6 +137,16 @@ def _launch(self, threads=None, solver=None, extract_data=True):
             )
         return resp
 
+    def check_progress(self):
+        """Get the status of an ongoing simulation, if possible
+
+        :return: (*dict*) -- json response
+        """
+        scenario_id = self.scenario.scenario_id
+        url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}"
+        resp = requests.get(url)
+        return resp.json()
+
 
 class NativeLauncher(Launcher):
     def _launch(self, threads=None, solver=None, extract_data=True):
@@ -137,3 +160,10 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :return: (*dict*) -- json response
         """
         pass
+
+    def check_progress(self):
+        """Get the status of an ongoing simulation, if possible
+
+        :return: (*dict*) -- json response
+        """
+        pass
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 7248786d0..e9fe4c41a 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -2,8 +2,6 @@
 import os
 import posixpath
 
-import requests
-
 from powersimdata.data_access.context import Context
 from powersimdata.input.case_mat import export_case_mat
 from powersimdata.input.grid import Grid
@@ -12,7 +10,7 @@
 from powersimdata.input.transform_profile import TransformProfile
 from powersimdata.scenario.state import State
 from powersimdata.utility import server_setup
-from powersimdata.utility.config import DeploymentMode, get_deployment_mode
+from powersimdata.utility.config import get_deployment_mode
 
 
 class Execute(State):
@@ -171,18 +169,11 @@ def launch_simulation(self, threads=None, extract_data=True, solver=None):
         self._launcher.launch_simulation(threads, extract_data, solver)
 
     def check_progress(self):
-        """Get the lastest information from the server container
+        """Get the status of an ongoing simulation, if possible
 
-        :raises NotImplementedError: if not running in container mode
+        :return: (*dict*) -- progress information, or None
         """
-        mode = get_deployment_mode()
-        if mode != DeploymentMode.Container:
-            raise NotImplementedError("Operation only supported for container mode")
-
-        scenario_id = self.scenario_id
-        url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}"
-        resp = requests.get(url)
-        return resp.json()
+        return self._launcher.check_progress()
 
     def extract_simulation_output(self):
         """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} on server.
@@ -192,13 +183,7 @@ def extract_simulation_output(self):
         """
         self._update_scenario_status()
         if self._scenario_status == "finished":
-            mode = get_deployment_mode()
-            if mode == DeploymentMode.Container:
-                print("WARNING: extraction not yet supported, please extract manually")
-                return
-
-            print("--> Extracting output data on server")
-            return self._run_script("extract_data.py")
+            return self._launcher.extract_simulation_output()
         else:
             print("---------------------------")
             print("OUTPUTS CANNOT BE EXTRACTED")

From 5a6e4971035e90edcc1b70a3e59f2b417a216ebe Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Wed, 5 May 2021 15:12:15 -0700
Subject: [PATCH 04/11] feat: launch simulation natively

---
 powersimdata/data_access/launcher.py | 11 +++++++++--
 powersimdata/scenario/execute.py     |  1 -
 powersimdata/utility/config.py       |  5 +++++
 powersimdata/utility/server_setup.py |  1 +
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index a6e77a137..68deb080b 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -1,4 +1,5 @@
 import posixpath
+import sys
 
 import requests
 
@@ -159,11 +160,17 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :param bool extract_data: always True
         :return: (*dict*) -- json response
         """
-        pass
+        sys.path.append(server_setup.ENGINE_DIR)
+        from pyreisejl.utility import app
+
+        return app.launch_simulation(self.scenario.scenario_id, threads, solver)
 
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
 
         :return: (*dict*) -- json response
         """
-        pass
+        sys.path.append(server_setup.ENGINE_DIR)
+        from pyreisejl.utility import app
+
+        return app.check_progress()
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index e9fe4c41a..4cbb50da5 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -1,6 +1,5 @@
 import copy
 import os
-import posixpath
 
 from powersimdata.data_access.context import Context
 from powersimdata.input.case_mat import export_case_mat
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index 498c3df54..b13a96187 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -5,6 +5,10 @@
 
 @dataclass(frozen=True)
 class Config:
+    SERVER_ADDRESS = None
+    SERVER_SSH_PORT = None
+    BACKUP_DATA_ROOT_DIR = None
+    ENGINE_DIR = None
     DATA_ROOT_DIR = "/mnt/bes/pcm"
     EXECUTE_DIR = "tmp"
     INPUT_DIR = ("data", "input")
@@ -28,6 +32,7 @@ class ContainerConfig(Config):
 @dataclass(frozen=True)
 class LocalConfig(Config):
     DATA_ROOT_DIR = Config.LOCAL_DIR
+    ENGINE_DIR = os.getenv("ENGINE_DIR")
 
 
 class DeploymentMode:
diff --git a/powersimdata/utility/server_setup.py b/powersimdata/utility/server_setup.py
index 55d7f84ad..e528e5c2d 100644
--- a/powersimdata/utility/server_setup.py
+++ b/powersimdata/utility/server_setup.py
@@ -12,6 +12,7 @@
 OUTPUT_DIR = config.OUTPUT_DIR
 LOCAL_DIR = config.LOCAL_DIR
 MODEL_DIR = config.MODEL_DIR
+ENGINE_DIR = config.ENGINE_DIR
 
 
 def get_server_user():

From 3db6dff3542c3147b93d1c32cefb25fd9bfffe10 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 6 May 2021 12:15:56 -0700
Subject: [PATCH 05/11] fix: method signature, docstrings

---
 powersimdata/data_access/launcher.py | 14 +++++++++-----
 powersimdata/scenario/execute.py     |  8 ++++----
 powersimdata/utility/config.py       |  4 ++--
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index 68deb080b..b821957ba 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -45,7 +45,7 @@ def extract_simulation_output(self):
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
         _check_threads(threads)
         _check_solver(solver)
-        self._launch(threads, solver, extract_data)
+        return self._launch(threads, solver, extract_data)
 
 
 class SSHLauncher(Launcher):
@@ -127,7 +127,8 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
         :param bool extract_data: always True
-        :return: (*requests.Response*) -- the http response object
+        :return: (*requests.Response*) -- http response from the engine, with a json
+            body as is returned by check_progress
         """
         scenario_id = self.scenario.scenario_id
         url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}"
@@ -141,7 +142,8 @@ def _launch(self, threads=None, solver=None, extract_data=True):
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
 
-        :return: (*dict*) -- json response
+        :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status"
+            keys which map to stdout, stderr, and the respective scenario attributes
         """
         scenario_id = self.scenario.scenario_id
         url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}"
@@ -158,7 +160,8 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
         :param bool extract_data: always True
-        :return: (*dict*) -- json response
+        :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status"
+            keys which map to stdout, stderr, and the respective scenario attributes
         """
         sys.path.append(server_setup.ENGINE_DIR)
         from pyreisejl.utility import app
@@ -168,7 +171,8 @@ def _launch(self, threads=None, solver=None, extract_data=True):
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
 
-        :return: (*dict*) -- json response
+        :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status"
+            keys which map to stdout, stderr, and the respective scenario attributes
         """
         sys.path.append(server_setup.ENGINE_DIR)
         from pyreisejl.utility import app
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 4cbb50da5..6df3b2f04 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -149,15 +149,15 @@ def _check_if_ready(self):
                 f"Status must be one of {valid_status}, but got status={self._scenario_status}"
             )
 
-    def launch_simulation(self, threads=None, extract_data=True, solver=None):
+    def launch_simulation(self, threads=None, solver=None, extract_data=True):
         """Launches simulation on target environment (server or container)
 
         :param int/None threads: the number of threads to be used. This defaults to None,
             where None means auto.
-        :param bool extract_data: whether the results of the simulation engine should
-            automatically extracted after the simulation has run. This defaults to True.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
+        :param bool extract_data: whether the results of the simulation engine should
+            automatically extracted after the simulation has run. This defaults to True.
         :return: (*subprocess.Popen*) or (*requests.Response*) - either the
             process (if using ssh to server) or http response (if run in container)
         """
@@ -165,7 +165,7 @@ def launch_simulation(self, threads=None, extract_data=True, solver=None):
 
         mode = get_deployment_mode()
         print(f"--> Launching simulation on {mode.lower()}")
-        self._launcher.launch_simulation(threads, extract_data, solver)
+        return self._launcher.launch_simulation(threads, solver, extract_data)
 
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index b13a96187..aa014453e 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -8,13 +8,13 @@ class Config:
     SERVER_ADDRESS = None
     SERVER_SSH_PORT = None
     BACKUP_DATA_ROOT_DIR = None
+    MODEL_DIR = None
     ENGINE_DIR = None
     DATA_ROOT_DIR = "/mnt/bes/pcm"
     EXECUTE_DIR = "tmp"
     INPUT_DIR = ("data", "input")
     OUTPUT_DIR = ("data", "output")
     LOCAL_DIR = os.path.join(Path.home(), "ScenarioData", "")
-    MODEL_DIR = "/home/bes/pcm"
 
 
 @dataclass(frozen=True)
@@ -22,6 +22,7 @@ class ServerConfig(Config):
     SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
     SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
     BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2"
+    MODEL_DIR = "/home/bes/pcm"
 
 
 @dataclass(frozen=True)
@@ -44,7 +45,6 @@ class DeploymentMode:
 
 
 def get_deployment_mode():
-    # TODO: consider auto detection
     mode = os.getenv("DEPLOYMENT_MODE")
     if mode is None:
         return DeploymentMode.Server

From 1abe2b5aa43da0b35286e00b2da5bd5625a24e36 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 6 May 2021 15:36:57 -0700
Subject: [PATCH 06/11] fix: refresh attributes on state change

---
 powersimdata/scenario/analyze.py  | 10 +++++++---
 powersimdata/scenario/create.py   |  6 +++---
 powersimdata/scenario/delete.py   |  1 -
 powersimdata/scenario/execute.py  | 17 +++++++++--------
 powersimdata/scenario/move.py     |  1 -
 powersimdata/scenario/scenario.py |  3 ++-
 powersimdata/scenario/state.py    | 21 ++++++++++-----------
 7 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/powersimdata/scenario/analyze.py b/powersimdata/scenario/analyze.py
index 950896bf8..e299a80db 100644
--- a/powersimdata/scenario/analyze.py
+++ b/powersimdata/scenario/analyze.py
@@ -39,17 +39,16 @@ class Analyze(State):
         "get_storage_pg",
         "get_wind",
         "print_infeasibilities",
-        "print_scenario_info",
     }
 
     def __init__(self, scenario):
         """Constructor."""
-        self._scenario_info = scenario.info
-        self._scenario_status = scenario.status
         super().__init__(scenario)
 
         self.data_loc = "disk" if scenario.status == "moved" else None
+        self.refresh(scenario)
 
+    def refresh(self, scenario):
         print(
             "SCENARIO: %s | %s\n"
             % (self._scenario_info["plan"], self._scenario_info["name"])
@@ -321,3 +320,8 @@ def get_wind(self):
         """
         profile = TransformProfile(self._scenario_info, self.get_grid(), self.get_ct())
         return profile.get_profile("wind")
+
+    def _leave(self):
+        """Cleans when leaving state."""
+        del self.grid
+        del self.ct
diff --git a/powersimdata/scenario/create.py b/powersimdata/scenario/create.py
index bc0b677b4..67bac22a9 100644
--- a/powersimdata/scenario/create.py
+++ b/powersimdata/scenario/create.py
@@ -27,7 +27,6 @@ class Create(State):
     default_exported_methods = (
         "create_scenario",
         "get_bus_demand",
-        "print_scenario_info",
         "set_builder",
         "set_grid",
     )
@@ -37,8 +36,6 @@ def __init__(self, scenario):
         self.builder = None
         self.grid = None
         self.ct = None
-        self._scenario_status = None
-        self._scenario_info = scenario.info
         self.exported_methods = set(self.default_exported_methods)
         super().__init__(scenario)
 
@@ -176,6 +173,9 @@ def set_grid(self, grid_model="usa_tamu", interconnect="USA"):
         self._scenario_info["grid_model"] = self.builder.grid_model
         self._scenario_info["interconnect"] = self.builder.interconnect
 
+    def _leave(self):
+        del self.builder
+
 
 class _Builder(object):
     """Scenario Builder.
diff --git a/powersimdata/scenario/delete.py b/powersimdata/scenario/delete.py
index 7a9587a0e..db80e6dab 100644
--- a/powersimdata/scenario/delete.py
+++ b/powersimdata/scenario/delete.py
@@ -10,7 +10,6 @@ class Delete(State):
     allowed = []
     exported_methods = {
         "delete_scenario",
-        "print_scenario_info",
     }
 
     def print_scenario_info(self):
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 6df3b2f04..b9bf0b46a 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -27,17 +27,20 @@ class Execute(State):
         "get_grid",
         "launch_simulation",
         "prepare_simulation_input",
-        "print_scenario_info",
         "print_scenario_status",
         "scenario_id",
     }
 
     def __init__(self, scenario):
         """Constructor."""
-        self._scenario_info = scenario.info
-        self._scenario_status = scenario.status
         super().__init__(scenario)
+        self.refresh(scenario)
 
+    @property
+    def scenario_id(self):
+        return self._scenario_info["id"]
+
+    def refresh(self, scenario):
         print(
             "SCENARIO: %s | %s\n"
             % (self._scenario_info["plan"], self._scenario_info["name"])
@@ -48,10 +51,6 @@ def __init__(self, scenario):
         self._set_ct_and_grid()
         self._launcher = Context.get_launcher(scenario)
 
-    @property
-    def scenario_id(self):
-        return self._scenario_info["id"]
-
     def _set_ct_and_grid(self):
         """Sets change table and grid."""
         base_grid = Grid(
@@ -129,7 +128,9 @@ def prepare_simulation_input(self, profiles_as=None):
 
             si.prepare_mpc_file()
 
-            self._execute_list_manager.set_status(self.scenario_id, "prepared")
+            prepared = "prepared"
+            self._execute_list_manager.set_status(self.scenario_id, prepared)
+            self._scenario_status = prepared
         else:
             print("---------------------------")
             print("SCENARIO CANNOT BE PREPARED")
diff --git a/powersimdata/scenario/move.py b/powersimdata/scenario/move.py
index d811842ba..374ec1542 100644
--- a/powersimdata/scenario/move.py
+++ b/powersimdata/scenario/move.py
@@ -11,7 +11,6 @@ class Move(State):
     allowed = []
     exported_methods = {
         "move_scenario",
-        "print_scenario_info",
     }
 
     def print_scenario_info(self):
diff --git a/powersimdata/scenario/scenario.py b/powersimdata/scenario/scenario.py
index 63e804b5e..fdc27b27b 100644
--- a/powersimdata/scenario/scenario.py
+++ b/powersimdata/scenario/scenario.py
@@ -57,6 +57,7 @@ def __init__(self, descriptor=None):
 
         if not descriptor:
             self.info = OrderedDict(self._default_info)
+            self.status = None
             self.state = Create(self)
         else:
             self._set_info(descriptor)
@@ -68,7 +69,7 @@ def __init__(self, descriptor=None):
                 elif state == "analyze":
                     self.state = Analyze(self)
             except AttributeError:
-                return
+                pass
 
     def __getattr__(self, name):
         if name in self.state.exported_methods:
diff --git a/powersimdata/scenario/state.py b/powersimdata/scenario/state.py
index 5bfcff997..944103f16 100644
--- a/powersimdata/scenario/state.py
+++ b/powersimdata/scenario/state.py
@@ -1,7 +1,3 @@
-from powersimdata.data_access.execute_list import ExecuteListManager
-from powersimdata.data_access.scenario_list import ScenarioListManager
-
-
 class State(object):
     """Defines an interface for encapsulating the behavior associated with a
         particular state of the Scenario object.
@@ -18,9 +14,15 @@ def __init__(self, scenario):
         if type(self) == State:
             raise TypeError("Only subclasses of 'State' can be instantiated directly")
 
+        self._scenario = scenario
+        self._scenario_info = scenario.info
+        self._scenario_status = scenario.status
         self._data_access = scenario.data_access
-        self._scenario_list_manager = ScenarioListManager(self._data_access)
-        self._execute_list_manager = ExecuteListManager(self._data_access)
+        self._scenario_list_manager = scenario._scenario_list_manager
+        self._execute_list_manager = scenario._execute_list_manager
+
+    def refresh(self, scenario):
+        pass
 
     def switch(self, state):
         """Switches state.
@@ -33,6 +35,7 @@ def switch(self, state):
             self._leave()
             self.__class__ = state
             self._enter(state)
+            self.refresh(self._scenario)
         else:
             raise Exception(
                 "State switching: %s --> %s not permitted" % (self, state.name)
@@ -47,11 +50,7 @@ def __str__(self):
 
     def _leave(self):
         """Cleans when leaving state."""
-        if self.name == "create":
-            del self.builder
-        elif self.name == "analyze":
-            del self.grid
-            del self.ct
+        pass
 
     def _enter(self, state):
         """Initializes when entering state."""

From 179e9ca68658318d82e5e9f6b9bf8c6f09234cae Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Fri, 7 May 2021 15:28:23 -0700
Subject: [PATCH 07/11] chore: address PR comments

---
 powersimdata/data_access/context.py           |  5 +++-
 powersimdata/data_access/data_access.py       | 10 ++++----
 powersimdata/data_access/launcher.py          | 24 ++++++++++++++++++-
 .../data_access/tests/test_data_access.py     |  4 ++--
 powersimdata/scenario/analyze.py              |  4 +++-
 powersimdata/scenario/execute.py              |  3 ++-
 powersimdata/utility/config.py                |  1 -
 7 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/powersimdata/data_access/context.py b/powersimdata/data_access/context.py
index afb7caa38..88ad74d47 100644
--- a/powersimdata/data_access/context.py
+++ b/powersimdata/data_access/context.py
@@ -14,6 +14,8 @@ def get_data_access(data_loc=None):
 
         :param str data_loc: pass "disk" if using data from backup disk,
             otherwise leave the default.
+        :return: (:class:`powersimdata.data_access.data_access.DataAccess`) -- a data access
+            instance
         """
         if data_loc == "disk":
             root = server_setup.BACKUP_DATA_ROOT_DIR
@@ -29,7 +31,8 @@ def get_data_access(data_loc=None):
     def get_launcher(scenario):
         """Return instance for interaction with simulation engine
 
-        :param powersimdata.Scenario scenario: a scenario object
+        :param powersimdata.scenario.scenario.Scenario scenario: a scenario object
+        :return: (:class:`powersimdata.data_access.launcher.Launcher`) -- a launcher instance
         """
         mode = get_deployment_mode()
         if mode == DeploymentMode.Server:
diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py
index 0d900b7ca..be854f14d 100644
--- a/powersimdata/data_access/data_access.py
+++ b/powersimdata/data_access/data_access.py
@@ -24,10 +24,12 @@
 class DataAccess:
     """Interface to a local or remote data store."""
 
-    def __init__(self, root=None):
+    def __init__(self, root=None, backup_root=None):
         """Constructor"""
         self.root = server_setup.DATA_ROOT_DIR if root is None else root
-        self.backup_root = server_setup.BACKUP_DATA_ROOT_DIR
+        self.backup_root = (
+            server_setup.BACKUP_DATA_ROOT_DIR if backup_root is None else backup_root
+        )
         self.join = None
 
     def copy_from(self, file_name, from_dir):
@@ -291,9 +293,9 @@ class SSHDataAccess(DataAccess):
 
     _last_attempt = 0
 
-    def __init__(self, root=None):
+    def __init__(self, root=None, backup_root=None):
         """Constructor"""
-        super().__init__(root)
+        super().__init__(root, backup_root)
         self._ssh = None
         self._retry_after = 5
         self.local_root = server_setup.LOCAL_DIR
diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index b821957ba..e2645afb2 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -36,6 +36,16 @@ def __init__(self, scenario):
         self.scenario = scenario
 
     def _launch(self, threads=None, solver=None, extract_data=True):
+        """Launches simulation on target environment
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: whether the results of the simulation engine should
+            automatically extracted after the simulation has run. This defaults to True.
+        :raises NotImplementedError: always - this must be implemented in a subclass
+        """
         raise NotImplementedError
 
     def extract_simulation_output(self):
@@ -43,6 +53,18 @@ def extract_simulation_output(self):
         pass
 
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
+        """Launches simulation on target environment
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: whether the results of the simulation engine should
+            automatically extracted after the simulation has run. This defaults to True.
+        :return: (*subprocess.Popen*) or (*requests.Response*) - either the
+            process (if using ssh to server) or http response (if run in container)
+            or (*dict*) (if run locally)
+        """
         _check_threads(threads)
         _check_solver(solver)
         return self._launch(threads, solver, extract_data)
@@ -99,7 +121,7 @@ def _launch(self, threads=None, solver=None, extract_data=True):
             extra_args.append("--solver " + solver)
 
         if not isinstance(extract_data, bool):
-            raise TypeError("extract_data must be a boolean: 'True' or 'False'")
+            raise TypeError("extract_data must be a bool")
         if extract_data:
             extra_args.append("--extract-data")
 
diff --git a/powersimdata/data_access/tests/test_data_access.py b/powersimdata/data_access/tests/test_data_access.py
index 7a2027c32..f29bcbe02 100644
--- a/powersimdata/data_access/tests/test_data_access.py
+++ b/powersimdata/data_access/tests/test_data_access.py
@@ -10,11 +10,12 @@
 from powersimdata.utility import server_setup
 
 CONTENT = b"content"
+backup_root = "/mnt/backup_dir"
 
 
 @pytest.fixture
 def data_access():
-    data_access = SSHDataAccess()
+    data_access = SSHDataAccess(backup_root=backup_root)
     yield data_access
     data_access.close()
 
@@ -68,7 +69,6 @@ def _check_content(filepath):
 
 
 root_dir = server_setup.DATA_ROOT_DIR.rstrip("/")
-backup_root = server_setup.BACKUP_DATA_ROOT_DIR
 
 
 def test_base_dir(data_access):
diff --git a/powersimdata/scenario/analyze.py b/powersimdata/scenario/analyze.py
index e299a80db..c29268d0e 100644
--- a/powersimdata/scenario/analyze.py
+++ b/powersimdata/scenario/analyze.py
@@ -61,7 +61,9 @@ def refresh(self, scenario):
     def _set_allowed_state(self):
         """Sets allowed state."""
         if self._scenario_status == "extracted":
-            self.allowed = ["delete", "move"]
+            self.allowed = ["delete"]
+            if self._data_access.backup_root is not None:
+                self.allowed.append("move")
 
     def _set_ct_and_grid(self):
         """Sets change table and grid."""
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index b9bf0b46a..a56add4f7 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -151,7 +151,7 @@ def _check_if_ready(self):
             )
 
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
-        """Launches simulation on target environment (server or container)
+        """Launches simulation on target environment
 
         :param int/None threads: the number of threads to be used. This defaults to None,
             where None means auto.
@@ -161,6 +161,7 @@ def launch_simulation(self, threads=None, solver=None, extract_data=True):
             automatically extracted after the simulation has run. This defaults to True.
         :return: (*subprocess.Popen*) or (*requests.Response*) - either the
             process (if using ssh to server) or http response (if run in container)
+            or (*dict*) (if run locally)
         """
         self._check_if_ready()
 
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index aa014453e..20de76fb0 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -21,7 +21,6 @@ class Config:
 class ServerConfig(Config):
     SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
     SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
-    BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2"
     MODEL_DIR = "/home/bes/pcm"
 
 

From 3f04222105351c81d88ec23e5c0a4b3388168484 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Wed, 12 May 2021 18:17:22 -0700
Subject: [PATCH 08/11] feat, docs: provision local directory and fix
 docstrings

---
 powersimdata/data_access/launcher.py |  5 ++++
 powersimdata/scenario/create.py      |  1 +
 powersimdata/scenario/execute.py     |  8 ++++++
 powersimdata/scenario/state.py       |  6 ++++-
 powersimdata/utility/config.py       | 38 ++++++++++++++++++++++++++++
 5 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index e2645afb2..59b2e8a2c 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -32,6 +32,11 @@ def _check_solver(solver):
 
 
 class Launcher:
+    """Base class for interaction with simulation engine.
+
+    :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance
+    """
+
     def __init__(self, scenario):
         self.scenario = scenario
 
diff --git a/powersimdata/scenario/create.py b/powersimdata/scenario/create.py
index 67bac22a9..a730638b6 100644
--- a/powersimdata/scenario/create.py
+++ b/powersimdata/scenario/create.py
@@ -174,6 +174,7 @@ def set_grid(self, grid_model="usa_tamu", interconnect="USA"):
         self._scenario_info["interconnect"] = self.builder.interconnect
 
     def _leave(self):
+        """Cleans when leaving state."""
         del self.builder
 
 
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index a56add4f7..1019b677f 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -38,9 +38,17 @@ def __init__(self, scenario):
 
     @property
     def scenario_id(self):
+        """Get the current scenario id
+
+        :return: (*str*) -- scenario id
+        """
         return self._scenario_info["id"]
 
     def refresh(self, scenario):
+        """Called during state changes to ensure instance is properly initialized
+
+        :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance
+        """
         print(
             "SCENARIO: %s | %s\n"
             % (self._scenario_info["plan"], self._scenario_info["name"])
diff --git a/powersimdata/scenario/state.py b/powersimdata/scenario/state.py
index 944103f16..cfd3d8e72 100644
--- a/powersimdata/scenario/state.py
+++ b/powersimdata/scenario/state.py
@@ -1,6 +1,6 @@
 class State(object):
     """Defines an interface for encapsulating the behavior associated with a
-        particular state of the Scenario object.
+    particular state of the Scenario object.
 
     :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance
     :raise TypeError: if not instantiated through a derived class
@@ -22,6 +22,10 @@ def __init__(self, scenario):
         self._execute_list_manager = scenario._execute_list_manager
 
     def refresh(self, scenario):
+        """Called during state changes to ensure instance is properly initialized
+
+        :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance
+        """
         pass
 
     def switch(self, state):
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index 20de76fb0..2f7a0f6e7 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -1,10 +1,17 @@
 import os
+import shutil
 from dataclasses import dataclass
 from pathlib import Path
 
+from powersimdata.utility import templates
+
 
 @dataclass(frozen=True)
 class Config:
+    """Base class for configuration data. It should contain all expected keys,
+    defaulting to None when not universally applicable.
+    """
+
     SERVER_ADDRESS = None
     SERVER_SSH_PORT = None
     BACKUP_DATA_ROOT_DIR = None
@@ -19,6 +26,8 @@ class Config:
 
 @dataclass(frozen=True)
 class ServerConfig(Config):
+    """Values specific to internal client/server usage"""
+
     SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
     SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
     MODEL_DIR = "/home/bes/pcm"
@@ -26,16 +35,37 @@ class ServerConfig(Config):
 
 @dataclass(frozen=True)
 class ContainerConfig(Config):
+    """Values specific to containerized environment"""
+
     SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "reisejl")
 
 
 @dataclass(frozen=True)
 class LocalConfig(Config):
+    """Values specific to native installation"""
+
     DATA_ROOT_DIR = Config.LOCAL_DIR
     ENGINE_DIR = os.getenv("ENGINE_DIR")
 
+    def initialize(self):
+        """Create data directory with blank templates"""
+        confirmed = input(
+            f"Provision directory {self.LOCAL_DIR}? [y/n] (default is 'n')"
+        )
+        if confirmed.lower() != "y":
+            print("Operation cancelled.")
+            return
+        os.makedirs(self.LOCAL_DIR, exist_ok=True)
+        for fname in ("ScenarioList.csv", "ExecuteList.csv"):
+            orig = os.path.join(templates.__path__[0], fname)
+            dest = os.path.join(self.LOCAL_DIR, fname)
+            shutil.copy(orig, dest)
+        print("--> Done!")
+
 
 class DeploymentMode:
+    """Constants representing the type of installation being used"""
+
     Server = "SERVER"
     Container = "CONTAINER"
     Local = "LOCAL"
@@ -44,6 +74,10 @@ class DeploymentMode:
 
 
 def get_deployment_mode():
+    """Get the deployment mode used to determine various configuration values
+
+    :return: (*str*) -- the deployment mode
+    """
     mode = os.getenv("DEPLOYMENT_MODE")
     if mode is None:
         return DeploymentMode.Server
@@ -54,5 +88,9 @@ def get_deployment_mode():
 
 
 def get_config():
+    """Get a config instance based on the DEPLOYMENT_MODE environment variable
+
+    :return: (*powersimdata.utility.config.Config*) -- a config instance
+    """
     mode = get_deployment_mode()
     return DeploymentMode.CONFIG_MAP[mode]()

From 086022783d7aeb79d3f8fa0ec4426746058118ed Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 13 May 2021 11:59:14 -0700
Subject: [PATCH 09/11] feat: configure root dir using config file

---
 .dockerignore                  | 4 ++++
 .gitignore                     | 1 +
 powersimdata/utility/config.py | 8 ++++++++
 3 files changed, 13 insertions(+)

diff --git a/.dockerignore b/.dockerignore
index bf5e8d691..192d301d8 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -7,3 +7,7 @@ build
 **/__pycache__
 .ipynb_checkpoints
 **/.ropeproject
+.env
+.venv
+.dockerignore
+config.ini
diff --git a/.gitignore b/.gitignore
index cbe3bcd07..01ba1505f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 # This is specific to this package
 powersimdata/utility/.server_user
+config.ini
 
 # The remainder of this file taken from github/gitignore
 # https://github.com/github/gitignore/blob/master/Python.gitignore
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index 2f7a0f6e7..9912d30b6 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -1,3 +1,4 @@
+import configparser
 import os
 import shutil
 from dataclasses import dataclass
@@ -5,6 +6,13 @@
 
 from powersimdata.utility import templates
 
+INI_FILE = "config.ini"
+if Path(INI_FILE).exists():
+    conf = configparser.ConfigParser()
+    conf.read(INI_FILE)
+    for k, v in conf["PowerSimData"].items():
+        os.environ[k.upper()] = v
+
 
 @dataclass(frozen=True)
 class Config:

From c84cacbdd8451c16b4952ee87e8a2262664ce06e Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Fri, 14 May 2021 10:54:15 -0700
Subject: [PATCH 10/11] fix: print statements mentioning the server

---
 powersimdata/data_access/execute_list.py  | 8 ++++----
 powersimdata/data_access/scenario_list.py | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/powersimdata/data_access/execute_list.py b/powersimdata/data_access/execute_list.py
index 1e70fbd88..5fa5c9b5d 100644
--- a/powersimdata/data_access/execute_list.py
+++ b/powersimdata/data_access/execute_list.py
@@ -18,7 +18,7 @@ def get_status(self, scenario_id):
         """Return the status for the scenario
 
         :param str/int scenario_id: the scenario id
-        :raises Exception: if scenario not found in execute list on server.
+        :raises Exception: if scenario not found in execute list.
         :return: (*str*) -- scenario status
         """
         table = self.get_execute_table()
@@ -46,12 +46,12 @@ def set_status(self, scenario_id, status):
         table = self.get_execute_table()
         table.loc[int(scenario_id), "status"] = status
 
-        print(f"-->  Setting status={status} in execute table on server")
+        print(f"--> Setting status={status} in execute list")
         return table
 
     @verify_hash
     def delete_entry(self, scenario_id):
-        """Deletes entry from execute list on server.
+        """Deletes entry from execute list.
 
         :param int/str scenario_id: the id of the scenario
         :return: (*pandas.DataFrame*) -- the updated data frame
@@ -59,5 +59,5 @@ def delete_entry(self, scenario_id):
         table = self.get_execute_table()
         table.drop(int(scenario_id), inplace=True)
 
-        print("--> Deleting entry in execute table on server")
+        print("--> Deleting entry in %s" % self._FILE_NAME)
         return table
diff --git a/powersimdata/data_access/scenario_list.py b/powersimdata/data_access/scenario_list.py
index 084817712..d67bcef07 100644
--- a/powersimdata/data_access/scenario_list.py
+++ b/powersimdata/data_access/scenario_list.py
@@ -64,7 +64,7 @@ def err_message(text):
 
     @verify_hash
     def add_entry(self, scenario_info):
-        """Adds scenario to the scenario list file on server.
+        """Adds scenario to the scenario list file.
 
         :param collections.OrderedDict scenario_info: entry to add to scenario list.
         :return: (*pandas.DataFrame*) -- the updated data frame
@@ -78,7 +78,7 @@ def add_entry(self, scenario_info):
         table = table.append(entry)
         table.set_index("id", inplace=True)
 
-        print("--> Adding entry in %s on server" % self._FILE_NAME)
+        print("--> Adding entry in %s" % self._FILE_NAME)
         return table
 
     @verify_hash
@@ -91,5 +91,5 @@ def delete_entry(self, scenario_id):
         table = self.get_scenario_table()
         table.drop(int(scenario_id), inplace=True)
 
-        print("--> Deleting entry in %s on server" % self._FILE_NAME)
+        print("--> Deleting entry in %s" % self._FILE_NAME)
         return table

From 27c0c560f81a3ee41533fd3dbd63d7a1e009c1d7 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Tue, 18 May 2021 15:44:22 -0700
Subject: [PATCH 11/11] chore: improve docstrings and consolidate return type
 for non ssh launchers

---
 powersimdata/data_access/launcher.py | 30 +++++++++++++++-------------
 powersimdata/scenario/execute.py     |  9 +++++----
 2 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index 59b2e8a2c..10a423d26 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -13,7 +13,7 @@ def _check_threads(threads):
     :raises TypeError: if threads is not an int
     :raises ValueError: if threads is not a positive value
     """
-    if threads:
+    if threads is not None:
         if not isinstance(threads, int):
             raise TypeError("threads must be an int")
         if threads < 1:
@@ -24,8 +24,11 @@ def _check_solver(solver):
     """Validate solver argument
 
     :param str solver: the solver used for the optimization
+    :raises TypeError: if solver is not a str
     :raises ValueError: if invalid solver provided
     """
+    if not isinstance(solver, str):
+        raise TypeError("solver must be a str")
     solvers = ("gurobi", "glpk")
     if solver is not None and solver.lower() not in solvers:
         raise ValueError(f"Invalid solver: options are {solvers}")
@@ -43,7 +46,7 @@ def __init__(self, scenario):
     def _launch(self, threads=None, solver=None, extract_data=True):
         """Launches simulation on target environment
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
@@ -60,15 +63,14 @@ def extract_simulation_output(self):
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
         """Launches simulation on target environment
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
         :param bool extract_data: whether the results of the simulation engine should
             automatically extracted after the simulation has run. This defaults to True.
-        :return: (*subprocess.Popen*) or (*requests.Response*) - either the
-            process (if using ssh to server) or http response (if run in container)
-            or (*dict*) (if run locally)
+        :return: (*subprocess.Popen*) or (*dict*) - the process, if using ssh to server,
+            otherwise a dict containing status information.
         """
         _check_threads(threads)
         _check_solver(solver)
@@ -108,7 +110,7 @@ def _run_script(self, script, extra_args=None):
     def _launch(self, threads=None, solver=None, extract_data=True):
         """Launch simulation on server, via ssh.
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
@@ -147,15 +149,15 @@ def check_progress(self):
 
 class HttpLauncher(Launcher):
     def _launch(self, threads=None, solver=None, extract_data=True):
-        """Launches simulation in container via http call
+        """Launch simulation in container via http call
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
         :param bool extract_data: always True
-        :return: (*requests.Response*) -- http response from the engine, with a json
-            body as is returned by check_progress
+        :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status"
+            keys which map to stdout, stderr, and the respective scenario attributes
         """
         scenario_id = self.scenario.scenario_id
         url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}"
@@ -164,7 +166,7 @@ def _launch(self, threads=None, solver=None, extract_data=True):
             print(
                 f"Failed to launch simulation: status={resp.status_code}. See response for details"
             )
-        return resp
+        return resp.json()
 
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
@@ -180,9 +182,9 @@ def check_progress(self):
 
 class NativeLauncher(Launcher):
     def _launch(self, threads=None, solver=None, extract_data=True):
-        """Launches simulation by importing from REISE.jl
+        """Launch simulation by importing from REISE.jl
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 1019b677f..ae3d68280 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -167,9 +167,8 @@ def launch_simulation(self, threads=None, solver=None, extract_data=True):
             None, which translates to gurobi
         :param bool extract_data: whether the results of the simulation engine should
             automatically extracted after the simulation has run. This defaults to True.
-        :return: (*subprocess.Popen*) or (*requests.Response*) - either the
-            process (if using ssh to server) or http response (if run in container)
-            or (*dict*) (if run locally)
+        :return: (*subprocess.Popen*) or (*dict*) - the process, if using ssh to server,
+            otherwise a dict containing status information.
         """
         self._check_if_ready()
 
@@ -180,7 +179,9 @@ def launch_simulation(self, threads=None, solver=None, extract_data=True):
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
 
-        :return: (*dict*) -- progress information, or None
+        :return: (*dict*) -- either None if using ssh, or a dict which contains
+            "output", "errors", "scenario_id", and "status" keys which map to
+            stdout, stderr, and the respective scenario attributes
         """
         return self._launcher.check_progress()