diff --git a/.dockerignore b/.dockerignore index bf5e8d691..192d301d8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,7 @@ build **/__pycache__ .ipynb_checkpoints **/.ropeproject +.env +.venv +.dockerignore +config.ini diff --git a/.gitignore b/.gitignore index cbe3bcd07..01ba1505f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # This is specific to this package powersimdata/utility/.server_user +config.ini # The remainder of this file taken from github/gitignore # https://github.com/github/gitignore/blob/master/Python.gitignore diff --git a/powersimdata/data_access/context.py b/powersimdata/data_access/context.py index e14dcfb67..88ad74d47 100644 --- a/powersimdata/data_access/context.py +++ b/powersimdata/data_access/context.py @@ -1,6 +1,7 @@ from powersimdata.data_access.data_access import LocalDataAccess, SSHDataAccess +from powersimdata.data_access.launcher import HttpLauncher, NativeLauncher, SSHLauncher from powersimdata.utility import server_setup -from powersimdata.utility.server_setup import DeploymentMode, get_deployment_mode +from powersimdata.utility.config import DeploymentMode, get_deployment_mode class Context: @@ -13,6 +14,8 @@ def get_data_access(data_loc=None): :param str data_loc: pass "disk" if using data from backup disk, otherwise leave the default. + :return: (:class:`powersimdata.data_access.data_access.DataAccess`) -- a data access + instance """ if data_loc == "disk": root = server_setup.BACKUP_DATA_ROOT_DIR @@ -23,3 +26,17 @@ def get_data_access(data_loc=None): if mode == DeploymentMode.Server: return SSHDataAccess(root) return LocalDataAccess(root) + + @staticmethod + def get_launcher(scenario): + """Return instance for interaction with simulation engine + + :param powersimdata.scenario.scenario.Scenario scenario: a scenario object + :return: (:class:`powersimdata.data_access.launcher.Launcher`) -- a launcher instance + """ + mode = get_deployment_mode() + if mode == DeploymentMode.Server: + return SSHLauncher(scenario) + elif mode == DeploymentMode.Container: + return HttpLauncher(scenario) + return NativeLauncher(scenario) diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py index 0d900b7ca..be854f14d 100644 --- a/powersimdata/data_access/data_access.py +++ b/powersimdata/data_access/data_access.py @@ -24,10 +24,12 @@ class DataAccess: """Interface to a local or remote data store.""" - def __init__(self, root=None): + def __init__(self, root=None, backup_root=None): """Constructor""" self.root = server_setup.DATA_ROOT_DIR if root is None else root - self.backup_root = server_setup.BACKUP_DATA_ROOT_DIR + self.backup_root = ( + server_setup.BACKUP_DATA_ROOT_DIR if backup_root is None else backup_root + ) self.join = None def copy_from(self, file_name, from_dir): @@ -291,9 +293,9 @@ class SSHDataAccess(DataAccess): _last_attempt = 0 - def __init__(self, root=None): + def __init__(self, root=None, backup_root=None): """Constructor""" - super().__init__(root) + super().__init__(root, backup_root) self._ssh = None self._retry_after = 5 self.local_root = server_setup.LOCAL_DIR diff --git a/powersimdata/data_access/execute_list.py b/powersimdata/data_access/execute_list.py index 1e70fbd88..5fa5c9b5d 100644 --- a/powersimdata/data_access/execute_list.py +++ b/powersimdata/data_access/execute_list.py @@ -18,7 +18,7 @@ def get_status(self, scenario_id): """Return the status for the scenario :param str/int scenario_id: the scenario id - :raises Exception: if scenario not found in execute list on server. + :raises Exception: if scenario not found in execute list. :return: (*str*) -- scenario status """ table = self.get_execute_table() @@ -46,12 +46,12 @@ def set_status(self, scenario_id, status): table = self.get_execute_table() table.loc[int(scenario_id), "status"] = status - print(f"--> Setting status={status} in execute table on server") + print(f"--> Setting status={status} in execute list") return table @verify_hash def delete_entry(self, scenario_id): - """Deletes entry from execute list on server. + """Deletes entry from execute list. :param int/str scenario_id: the id of the scenario :return: (*pandas.DataFrame*) -- the updated data frame @@ -59,5 +59,5 @@ def delete_entry(self, scenario_id): table = self.get_execute_table() table.drop(int(scenario_id), inplace=True) - print("--> Deleting entry in execute table on server") + print("--> Deleting entry in %s" % self._FILE_NAME) return table diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py new file mode 100644 index 000000000..10a423d26 --- /dev/null +++ b/powersimdata/data_access/launcher.py @@ -0,0 +1,209 @@ +import posixpath +import sys + +import requests + +from powersimdata.utility import server_setup + + +def _check_threads(threads): + """Validate threads argument + + :param int threads: the number of threads to be used + :raises TypeError: if threads is not an int + :raises ValueError: if threads is not a positive value + """ + if threads is not None: + if not isinstance(threads, int): + raise TypeError("threads must be an int") + if threads < 1: + raise ValueError("threads must be a positive value") + + +def _check_solver(solver): + """Validate solver argument + + :param str solver: the solver used for the optimization + :raises TypeError: if solver is not a str + :raises ValueError: if invalid solver provided + """ + if not isinstance(solver, str): + raise TypeError("solver must be a str") + solvers = ("gurobi", "glpk") + if solver is not None and solver.lower() not in solvers: + raise ValueError(f"Invalid solver: options are {solvers}") + + +class Launcher: + """Base class for interaction with simulation engine. + + :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance + """ + + def __init__(self, scenario): + self.scenario = scenario + + def _launch(self, threads=None, solver=None, extract_data=True): + """Launches simulation on target environment + + :param int threads: the number of threads to be used. This defaults to None, + where None means auto. + :param str solver: the solver used for optimization. This defaults to + None, which translates to gurobi + :param bool extract_data: whether the results of the simulation engine should + automatically extracted after the simulation has run. This defaults to True. + :raises NotImplementedError: always - this must be implemented in a subclass + """ + raise NotImplementedError + + def extract_simulation_output(self): + """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} on server.""" + pass + + def launch_simulation(self, threads=None, solver=None, extract_data=True): + """Launches simulation on target environment + + :param int threads: the number of threads to be used. This defaults to None, + where None means auto. + :param str solver: the solver used for optimization. This defaults to + None, which translates to gurobi + :param bool extract_data: whether the results of the simulation engine should + automatically extracted after the simulation has run. This defaults to True. + :return: (*subprocess.Popen*) or (*dict*) - the process, if using ssh to server, + otherwise a dict containing status information. + """ + _check_threads(threads) + _check_solver(solver) + return self._launch(threads, solver, extract_data) + + +class SSHLauncher(Launcher): + def _run_script(self, script, extra_args=None): + """Returns running process + + :param str script: script to be used. + :param list extra_args: list of strings to be passed after scenario id. + :return: (*subprocess.Popen*) -- process used to run script + """ + if extra_args is None: + extra_args = [] + + engine = self.scenario._scenario_info["engine"] + path_to_package = posixpath.join(server_setup.MODEL_DIR, engine) + folder = "pyreise" if engine == "REISE" else "pyreisejl" + + path_to_script = posixpath.join(path_to_package, folder, "utility", script) + cmd_pythonpath = [f'export PYTHONPATH="{path_to_package}:$PYTHONPATH";'] + cmd_pythoncall = [ + "nohup", + "python3", + "-u", + path_to_script, + self.scenario.scenario_id, + ] + cmd_io_redirect = ["/dev/null 2>&1 &"] + cmd = cmd_pythonpath + cmd_pythoncall + extra_args + cmd_io_redirect + process = self.scenario._data_access.execute_command_async(cmd) + print("PID: %s" % process.pid) + return process + + def _launch(self, threads=None, solver=None, extract_data=True): + """Launch simulation on server, via ssh. + + :param int threads: the number of threads to be used. This defaults to None, + where None means auto. + :param str solver: the solver used for optimization. This defaults to + None, which translates to gurobi + :param bool extract_data: whether the results of the simulation engine should + automatically extracted after the simulation has run. This defaults to True. + :raises TypeError: if extract_data is not a boolean + :return: (*subprocess.Popen*) -- new process used to launch simulation. + """ + extra_args = [] + if threads is not None: + # Use the -t flag as defined in call.py in REISE.jl + extra_args.append("--threads " + str(threads)) + + if solver is not None: + extra_args.append("--solver " + solver) + + if not isinstance(extract_data, bool): + raise TypeError("extract_data must be a bool") + if extract_data: + extra_args.append("--extract-data") + + return self._run_script("call.py", extra_args=extra_args) + + def extract_simulation_output(self): + """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} on server. + + :return: (*subprocess.Popen*) -- new process used to extract output + data. + """ + print("--> Extracting output data on server") + return self._run_script("extract_data.py") + + def check_progress(self): + print("Information is available on the server.") + + +class HttpLauncher(Launcher): + def _launch(self, threads=None, solver=None, extract_data=True): + """Launch simulation in container via http call + + :param int threads: the number of threads to be used. This defaults to None, + where None means auto. + :param str solver: the solver used for optimization. This defaults to + None, which translates to gurobi + :param bool extract_data: always True + :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" + keys which map to stdout, stderr, and the respective scenario attributes + """ + scenario_id = self.scenario.scenario_id + url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}" + resp = requests.post(url, params={"threads": threads, "solver": solver}) + if resp.status_code != 200: + print( + f"Failed to launch simulation: status={resp.status_code}. See response for details" + ) + return resp.json() + + def check_progress(self): + """Get the status of an ongoing simulation, if possible + + :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" + keys which map to stdout, stderr, and the respective scenario attributes + """ + scenario_id = self.scenario.scenario_id + url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}" + resp = requests.get(url) + return resp.json() + + +class NativeLauncher(Launcher): + def _launch(self, threads=None, solver=None, extract_data=True): + """Launch simulation by importing from REISE.jl + + :param int threads: the number of threads to be used. This defaults to None, + where None means auto. + :param str solver: the solver used for optimization. This defaults to + None, which translates to gurobi + :param bool extract_data: always True + :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" + keys which map to stdout, stderr, and the respective scenario attributes + """ + sys.path.append(server_setup.ENGINE_DIR) + from pyreisejl.utility import app + + return app.launch_simulation(self.scenario.scenario_id, threads, solver) + + def check_progress(self): + """Get the status of an ongoing simulation, if possible + + :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" + keys which map to stdout, stderr, and the respective scenario attributes + """ + sys.path.append(server_setup.ENGINE_DIR) + from pyreisejl.utility import app + + return app.check_progress() diff --git a/powersimdata/data_access/scenario_list.py b/powersimdata/data_access/scenario_list.py index 084817712..d67bcef07 100644 --- a/powersimdata/data_access/scenario_list.py +++ b/powersimdata/data_access/scenario_list.py @@ -64,7 +64,7 @@ def err_message(text): @verify_hash def add_entry(self, scenario_info): - """Adds scenario to the scenario list file on server. + """Adds scenario to the scenario list file. :param collections.OrderedDict scenario_info: entry to add to scenario list. :return: (*pandas.DataFrame*) -- the updated data frame @@ -78,7 +78,7 @@ def add_entry(self, scenario_info): table = table.append(entry) table.set_index("id", inplace=True) - print("--> Adding entry in %s on server" % self._FILE_NAME) + print("--> Adding entry in %s" % self._FILE_NAME) return table @verify_hash @@ -91,5 +91,5 @@ def delete_entry(self, scenario_id): table = self.get_scenario_table() table.drop(int(scenario_id), inplace=True) - print("--> Deleting entry in %s on server" % self._FILE_NAME) + print("--> Deleting entry in %s" % self._FILE_NAME) return table diff --git a/powersimdata/data_access/tests/test_data_access.py b/powersimdata/data_access/tests/test_data_access.py index 7a2027c32..f29bcbe02 100644 --- a/powersimdata/data_access/tests/test_data_access.py +++ b/powersimdata/data_access/tests/test_data_access.py @@ -10,11 +10,12 @@ from powersimdata.utility import server_setup CONTENT = b"content" +backup_root = "/mnt/backup_dir" @pytest.fixture def data_access(): - data_access = SSHDataAccess() + data_access = SSHDataAccess(backup_root=backup_root) yield data_access data_access.close() @@ -68,7 +69,6 @@ def _check_content(filepath): root_dir = server_setup.DATA_ROOT_DIR.rstrip("/") -backup_root = server_setup.BACKUP_DATA_ROOT_DIR def test_base_dir(data_access): diff --git a/powersimdata/scenario/analyze.py b/powersimdata/scenario/analyze.py index 950896bf8..c29268d0e 100644 --- a/powersimdata/scenario/analyze.py +++ b/powersimdata/scenario/analyze.py @@ -39,17 +39,16 @@ class Analyze(State): "get_storage_pg", "get_wind", "print_infeasibilities", - "print_scenario_info", } def __init__(self, scenario): """Constructor.""" - self._scenario_info = scenario.info - self._scenario_status = scenario.status super().__init__(scenario) self.data_loc = "disk" if scenario.status == "moved" else None + self.refresh(scenario) + def refresh(self, scenario): print( "SCENARIO: %s | %s\n" % (self._scenario_info["plan"], self._scenario_info["name"]) @@ -62,7 +61,9 @@ def __init__(self, scenario): def _set_allowed_state(self): """Sets allowed state.""" if self._scenario_status == "extracted": - self.allowed = ["delete", "move"] + self.allowed = ["delete"] + if self._data_access.backup_root is not None: + self.allowed.append("move") def _set_ct_and_grid(self): """Sets change table and grid.""" @@ -321,3 +322,8 @@ def get_wind(self): """ profile = TransformProfile(self._scenario_info, self.get_grid(), self.get_ct()) return profile.get_profile("wind") + + def _leave(self): + """Cleans when leaving state.""" + del self.grid + del self.ct diff --git a/powersimdata/scenario/create.py b/powersimdata/scenario/create.py index bc0b677b4..a730638b6 100644 --- a/powersimdata/scenario/create.py +++ b/powersimdata/scenario/create.py @@ -27,7 +27,6 @@ class Create(State): default_exported_methods = ( "create_scenario", "get_bus_demand", - "print_scenario_info", "set_builder", "set_grid", ) @@ -37,8 +36,6 @@ def __init__(self, scenario): self.builder = None self.grid = None self.ct = None - self._scenario_status = None - self._scenario_info = scenario.info self.exported_methods = set(self.default_exported_methods) super().__init__(scenario) @@ -176,6 +173,10 @@ def set_grid(self, grid_model="usa_tamu", interconnect="USA"): self._scenario_info["grid_model"] = self.builder.grid_model self._scenario_info["interconnect"] = self.builder.interconnect + def _leave(self): + """Cleans when leaving state.""" + del self.builder + class _Builder(object): """Scenario Builder. diff --git a/powersimdata/scenario/delete.py b/powersimdata/scenario/delete.py index 7a9587a0e..db80e6dab 100644 --- a/powersimdata/scenario/delete.py +++ b/powersimdata/scenario/delete.py @@ -10,7 +10,6 @@ class Delete(State): allowed = [] exported_methods = { "delete_scenario", - "print_scenario_info", } def print_scenario_info(self): diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py index 9a92e30bc..ae3d68280 100644 --- a/powersimdata/scenario/execute.py +++ b/powersimdata/scenario/execute.py @@ -1,9 +1,7 @@ import copy import os -import posixpath - -import requests +from powersimdata.data_access.context import Context from powersimdata.input.case_mat import export_case_mat from powersimdata.input.grid import Grid from powersimdata.input.input_data import InputData @@ -11,7 +9,7 @@ from powersimdata.input.transform_profile import TransformProfile from powersimdata.scenario.state import State from powersimdata.utility import server_setup -from powersimdata.utility.server_setup import DeploymentMode, get_deployment_mode +from powersimdata.utility.config import get_deployment_mode class Execute(State): @@ -29,16 +27,28 @@ class Execute(State): "get_grid", "launch_simulation", "prepare_simulation_input", - "print_scenario_info", "print_scenario_status", + "scenario_id", } def __init__(self, scenario): """Constructor.""" - self._scenario_info = scenario.info - self._scenario_status = scenario.status super().__init__(scenario) + self.refresh(scenario) + + @property + def scenario_id(self): + """Get the current scenario id + :return: (*str*) -- scenario id + """ + return self._scenario_info["id"] + + def refresh(self, scenario): + """Called during state changes to ensure instance is properly initialized + + :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance + """ print( "SCENARIO: %s | %s\n" % (self._scenario_info["plan"], self._scenario_info["name"]) @@ -47,9 +57,7 @@ def __init__(self, scenario): print("--> Status\n%s" % self._scenario_status) self._set_ct_and_grid() - - def _scenario_id(self): - return self._scenario_info["id"] + self._launcher = Context.get_launcher(scenario) def _set_ct_and_grid(self): """Sets change table and grid.""" @@ -81,50 +89,11 @@ def get_grid(self): def _update_scenario_status(self): """Updates scenario status.""" - self._scenario_status = self._execute_list_manager.get_status( - self._scenario_id() - ) + self._scenario_status = self._execute_list_manager.get_status(self.scenario_id) def _update_scenario_info(self): """Updates scenario information.""" - self._scenario_info = self._scenario_list_manager.get_scenario( - self._scenario_id() - ) - - def _run_script(self, script, extra_args=None): - """Returns running process - - :param str script: script to be used. - :param list extra_args: list of strings to be passed after scenario id. - :return: (*subprocess.Popen*) -- process used to run script - """ - - if not extra_args: - extra_args = [] - - path_to_package = posixpath.join( - server_setup.MODEL_DIR, self._scenario_info["engine"] - ) - - if self._scenario_info["engine"] == "REISE": - folder = "pyreise" - else: - folder = "pyreisejl" - - path_to_script = posixpath.join(path_to_package, folder, "utility", script) - cmd_pythonpath = [f'export PYTHONPATH="{path_to_package}:$PYTHONPATH";'] - cmd_pythoncall = [ - "nohup", - "python3", - "-u", - path_to_script, - self._scenario_info["id"], - ] - cmd_io_redirect = ["/dev/null 2>&1 &"] - cmd = cmd_pythonpath + cmd_pythoncall + extra_args + cmd_io_redirect - process = self._data_access.execute_command_async(cmd) - print("PID: %s" % process.pid) - return process + self._scenario_info = self._scenario_list_manager.get_scenario(self.scenario_id) def print_scenario_info(self): """Prints scenario information.""" @@ -167,7 +136,9 @@ def prepare_simulation_input(self, profiles_as=None): si.prepare_mpc_file() - self._execute_list_manager.set_status(self._scenario_id(), "prepared") + prepared = "prepared" + self._execute_list_manager.set_status(self.scenario_id, prepared) + self._scenario_status = prepared else: print("---------------------------") print("SCENARIO CANNOT BE PREPARED") @@ -187,110 +158,32 @@ def _check_if_ready(self): f"Status must be one of {valid_status}, but got status={self._scenario_status}" ) - def _launch_on_server(self, threads=None, solver=None, extract_data=True): - """Launch simulation on server, via ssh. - - :param int/None threads: the number of threads to be used. This defaults to None, - where None means auto. - :param str solver: the solver used for optimization. This defaults to - None, which translates to gurobi - :param bool extract_data: whether the results of the simulation engine should - automatically extracted after the simulation has run. This defaults to True. - :raises TypeError: if extract_data is not a boolean - :return: (*subprocess.Popen*) -- new process used to launch simulation. - """ - extra_args = [] - - if threads: - # Use the -t flag as defined in call.py in REISE.jl - extra_args.append("--threads " + str(threads)) - - if solver: - extra_args.append("--solver " + solver) - - if not isinstance(extract_data, bool): - raise TypeError("extract_data must be a boolean: 'True' or 'False'") - if extract_data: - extra_args.append("--extract-data") - - return self._run_script("call.py", extra_args=extra_args) - - def _launch_in_container(self, threads, solver): - """Launches simulation in container via http call + def launch_simulation(self, threads=None, solver=None, extract_data=True): + """Launches simulation on target environment :param int/None threads: the number of threads to be used. This defaults to None, where None means auto. :param str solver: the solver used for optimization. This defaults to None, which translates to gurobi - :return: (*requests.Response*) -- the http response object - """ - scenario_id = self._scenario_id() - url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}" - resp = requests.post(url, params={"threads": threads, "solver": solver}) - if resp.status_code != 200: - print( - f"Failed to launch simulation: status={resp.status_code}. See response for details" - ) - return resp - - def _check_threads(self, threads): - """Validate threads argument - - :param int threads: the number of threads to be used - :raises TypeError: if threads is not an int - :raises ValueError: if threads is not a positive value - """ - if threads: - if not isinstance(threads, int): - raise TypeError("threads must be an int") - if threads < 1: - raise ValueError("threads must be a positive value") - - def _check_solver(self, solver): - """Validate solver argument - - :param str solver: the solver used for the optimization - :raises ValueError: if invalid solver provided - """ - solvers = ("gurobi", "glpk") - if solver is not None and solver.lower() not in solvers: - raise ValueError(f"Invalid solver: options are {solvers}") - - def launch_simulation(self, threads=None, extract_data=True, solver=None): - """Launches simulation on target environment (server or container) - - :param int/None threads: the number of threads to be used. This defaults to None, - where None means auto. :param bool extract_data: whether the results of the simulation engine should automatically extracted after the simulation has run. This defaults to True. - :param str solver: the solver used for optimization. This defaults to - None, which translates to gurobi - :return: (*subprocess.Popen*) or (*requests.Response*) - either the - process (if using ssh to server) or http response (if run in container) + :return: (*subprocess.Popen*) or (*dict*) - the process, if using ssh to server, + otherwise a dict containing status information. """ self._check_if_ready() - self._check_threads(threads) - self._check_solver(solver) mode = get_deployment_mode() print(f"--> Launching simulation on {mode.lower()}") - if mode == DeploymentMode.Server: - return self._launch_on_server(threads, solver, extract_data) - return self._launch_in_container(threads, solver) + return self._launcher.launch_simulation(threads, solver, extract_data) def check_progress(self): - """Get the lastest information from the server container + """Get the status of an ongoing simulation, if possible - :raises NotImplementedError: if not running in container mode + :return: (*dict*) -- either None if using ssh, or a dict which contains + "output", "errors", "scenario_id", and "status" keys which map to + stdout, stderr, and the respective scenario attributes """ - mode = get_deployment_mode() - if mode != DeploymentMode.Container: - raise NotImplementedError("Operation only supported for container mode") - - scenario_id = self._scenario_id() - url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}" - resp = requests.get(url) - return resp.json() + return self._launcher.check_progress() def extract_simulation_output(self): """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} on server. @@ -300,13 +193,7 @@ def extract_simulation_output(self): """ self._update_scenario_status() if self._scenario_status == "finished": - mode = get_deployment_mode() - if mode == DeploymentMode.Container: - print("WARNING: extraction not yet supported, please extract manually") - return - - print("--> Extracting output data on server") - return self._run_script("extract_data.py") + return self._launcher.extract_simulation_output() else: print("---------------------------") print("OUTPUTS CANNOT BE EXTRACTED") diff --git a/powersimdata/scenario/move.py b/powersimdata/scenario/move.py index d811842ba..374ec1542 100644 --- a/powersimdata/scenario/move.py +++ b/powersimdata/scenario/move.py @@ -11,7 +11,6 @@ class Move(State): allowed = [] exported_methods = { "move_scenario", - "print_scenario_info", } def print_scenario_info(self): diff --git a/powersimdata/scenario/scenario.py b/powersimdata/scenario/scenario.py index 63e804b5e..fdc27b27b 100644 --- a/powersimdata/scenario/scenario.py +++ b/powersimdata/scenario/scenario.py @@ -57,6 +57,7 @@ def __init__(self, descriptor=None): if not descriptor: self.info = OrderedDict(self._default_info) + self.status = None self.state = Create(self) else: self._set_info(descriptor) @@ -68,7 +69,7 @@ def __init__(self, descriptor=None): elif state == "analyze": self.state = Analyze(self) except AttributeError: - return + pass def __getattr__(self, name): if name in self.state.exported_methods: diff --git a/powersimdata/scenario/state.py b/powersimdata/scenario/state.py index 5bfcff997..cfd3d8e72 100644 --- a/powersimdata/scenario/state.py +++ b/powersimdata/scenario/state.py @@ -1,10 +1,6 @@ -from powersimdata.data_access.execute_list import ExecuteListManager -from powersimdata.data_access.scenario_list import ScenarioListManager - - class State(object): """Defines an interface for encapsulating the behavior associated with a - particular state of the Scenario object. + particular state of the Scenario object. :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance :raise TypeError: if not instantiated through a derived class @@ -18,9 +14,19 @@ def __init__(self, scenario): if type(self) == State: raise TypeError("Only subclasses of 'State' can be instantiated directly") + self._scenario = scenario + self._scenario_info = scenario.info + self._scenario_status = scenario.status self._data_access = scenario.data_access - self._scenario_list_manager = ScenarioListManager(self._data_access) - self._execute_list_manager = ExecuteListManager(self._data_access) + self._scenario_list_manager = scenario._scenario_list_manager + self._execute_list_manager = scenario._execute_list_manager + + def refresh(self, scenario): + """Called during state changes to ensure instance is properly initialized + + :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance + """ + pass def switch(self, state): """Switches state. @@ -33,6 +39,7 @@ def switch(self, state): self._leave() self.__class__ = state self._enter(state) + self.refresh(self._scenario) else: raise Exception( "State switching: %s --> %s not permitted" % (self, state.name) @@ -47,11 +54,7 @@ def __str__(self): def _leave(self): """Cleans when leaving state.""" - if self.name == "create": - del self.builder - elif self.name == "analyze": - del self.grid - del self.ct + pass def _enter(self, state): """Initializes when entering state.""" diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py new file mode 100644 index 000000000..9912d30b6 --- /dev/null +++ b/powersimdata/utility/config.py @@ -0,0 +1,104 @@ +import configparser +import os +import shutil +from dataclasses import dataclass +from pathlib import Path + +from powersimdata.utility import templates + +INI_FILE = "config.ini" +if Path(INI_FILE).exists(): + conf = configparser.ConfigParser() + conf.read(INI_FILE) + for k, v in conf["PowerSimData"].items(): + os.environ[k.upper()] = v + + +@dataclass(frozen=True) +class Config: + """Base class for configuration data. It should contain all expected keys, + defaulting to None when not universally applicable. + """ + + SERVER_ADDRESS = None + SERVER_SSH_PORT = None + BACKUP_DATA_ROOT_DIR = None + MODEL_DIR = None + ENGINE_DIR = None + DATA_ROOT_DIR = "/mnt/bes/pcm" + EXECUTE_DIR = "tmp" + INPUT_DIR = ("data", "input") + OUTPUT_DIR = ("data", "output") + LOCAL_DIR = os.path.join(Path.home(), "ScenarioData", "") + + +@dataclass(frozen=True) +class ServerConfig(Config): + """Values specific to internal client/server usage""" + + SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com") + SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22) + MODEL_DIR = "/home/bes/pcm" + + +@dataclass(frozen=True) +class ContainerConfig(Config): + """Values specific to containerized environment""" + + SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "reisejl") + + +@dataclass(frozen=True) +class LocalConfig(Config): + """Values specific to native installation""" + + DATA_ROOT_DIR = Config.LOCAL_DIR + ENGINE_DIR = os.getenv("ENGINE_DIR") + + def initialize(self): + """Create data directory with blank templates""" + confirmed = input( + f"Provision directory {self.LOCAL_DIR}? [y/n] (default is 'n')" + ) + if confirmed.lower() != "y": + print("Operation cancelled.") + return + os.makedirs(self.LOCAL_DIR, exist_ok=True) + for fname in ("ScenarioList.csv", "ExecuteList.csv"): + orig = os.path.join(templates.__path__[0], fname) + dest = os.path.join(self.LOCAL_DIR, fname) + shutil.copy(orig, dest) + print("--> Done!") + + +class DeploymentMode: + """Constants representing the type of installation being used""" + + Server = "SERVER" + Container = "CONTAINER" + Local = "LOCAL" + + CONFIG_MAP = {Server: ServerConfig, Container: ContainerConfig, Local: LocalConfig} + + +def get_deployment_mode(): + """Get the deployment mode used to determine various configuration values + + :return: (*str*) -- the deployment mode + """ + mode = os.getenv("DEPLOYMENT_MODE") + if mode is None: + return DeploymentMode.Server + if mode == "1" or mode.lower() == "container": + return DeploymentMode.Container + if mode == "2" or mode.lower() == "local": + return DeploymentMode.Local + + +def get_config(): + """Get a config instance based on the DEPLOYMENT_MODE environment variable + + :return: (*powersimdata.utility.config.Config*) -- a config instance + """ + mode = get_deployment_mode() + return DeploymentMode.CONFIG_MAP[mode]() diff --git a/powersimdata/utility/server_setup.py b/powersimdata/utility/server_setup.py index e65a74e45..e528e5c2d 100644 --- a/powersimdata/utility/server_setup.py +++ b/powersimdata/utility/server_setup.py @@ -1,27 +1,18 @@ import os -from pathlib import Path -SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com") -SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22) -BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2" -DATA_ROOT_DIR = "/mnt/bes/pcm" -EXECUTE_DIR = "tmp" -INPUT_DIR = ("data", "input") -OUTPUT_DIR = ("data", "output") -LOCAL_DIR = os.path.join(Path.home(), "ScenarioData", "") -MODEL_DIR = "/home/bes/pcm" - - -class DeploymentMode: - Server = "SERVER" - Container = "CONTAINER" - - -def get_deployment_mode(): - mode = os.getenv("DEPLOYMENT_MODE") - if mode is None: - return DeploymentMode.Server - return DeploymentMode.Container +from powersimdata.utility.config import get_config + +config = get_config() +SERVER_ADDRESS = config.SERVER_ADDRESS +SERVER_SSH_PORT = config.SERVER_SSH_PORT +BACKUP_DATA_ROOT_DIR = config.BACKUP_DATA_ROOT_DIR +DATA_ROOT_DIR = config.DATA_ROOT_DIR +EXECUTE_DIR = config.EXECUTE_DIR +INPUT_DIR = config.INPUT_DIR +OUTPUT_DIR = config.OUTPUT_DIR +LOCAL_DIR = config.LOCAL_DIR +MODEL_DIR = config.MODEL_DIR +ENGINE_DIR = config.ENGINE_DIR def get_server_user():