From 67448d905e7a6a55539cfc44e4406fc783ca7c13 Mon Sep 17 00:00:00 2001 From: Jon Hagg Date: Thu, 15 Jul 2021 15:23:24 -0700 Subject: [PATCH 1/3] feat: support passing extract_data for container and local simulations --- powersimdata/data_access/launcher.py | 45 ++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py index 2f2a5384f..f34646f22 100644 --- a/powersimdata/data_access/launcher.py +++ b/powersimdata/data_access/launcher.py @@ -151,6 +151,8 @@ def check_progress(self): class HttpLauncher(Launcher): + BASE_URL = f"http://{server_setup.SERVER_ADDRESS}:5000" + def _launch(self, threads=None, solver=None, extract_data=True): """Launch simulation in container via http call @@ -163,14 +165,30 @@ def _launch(self, threads=None, solver=None, extract_data=True): keys which map to stdout, stderr, and the respective scenario attributes """ scenario_id = self.scenario.scenario_id - url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}" - resp = requests.post(url, params={"threads": threads, "solver": solver}) + url = f"{self.BASE_URL}/launch/{scenario_id}" + params = { + "threads": threads, + "solver": solver, + "extract-data": int(extract_data), + } + resp = requests.post(url, params=params) if resp.status_code != 200: print( f"Failed to launch simulation: status={resp.status_code}. See response for details" ) return resp.json() + def extract_simulation_output(self): + """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} + + :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" + keys which map to stdout, stderr, and the respective scenario attributes + """ + scenario_id = self.scenario.scenario_id + url = f"{self.BASE_URL}/extract/{scenario_id}" + resp = requests.post(url) + return resp.json() + def check_progress(self): """Get the status of an ongoing simulation, if possible @@ -178,12 +196,17 @@ def check_progress(self): keys which map to stdout, stderr, and the respective scenario attributes """ scenario_id = self.scenario.scenario_id - url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}" + url = f"{self.BASE_URL}/status/{scenario_id}" resp = requests.get(url) return resp.json() class NativeLauncher(Launcher): + def __init__(self, scenario): + sys.path.append(server_setup.ENGINE_DIR) + + super().__init__(scenario) + def _launch(self, threads=None, solver=None, extract_data=True): """Launch simulation by importing from REISE.jl @@ -195,10 +218,21 @@ def _launch(self, threads=None, solver=None, extract_data=True): :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" keys which map to stdout, stderr, and the respective scenario attributes """ - sys.path.append(server_setup.ENGINE_DIR) from pyreisejl.utility import app - return app.launch_simulation(self.scenario.scenario_id, threads, solver) + return app.launch_simulation( + self.scenario.scenario_id, threads, solver, extract_data + ) + + def extract_simulation_output(self): + """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} + + :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" + keys which map to stdout, stderr, and the respective scenario attributes + """ + from pyreisejl.utility import app + + return app.extract_scenario(self.scenario.scenario_id) def check_progress(self): """Get the status of an ongoing simulation, if possible @@ -206,7 +240,6 @@ def check_progress(self): :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" keys which map to stdout, stderr, and the respective scenario attributes """ - sys.path.append(server_setup.ENGINE_DIR) from pyreisejl.utility import app return app.check_progress() From dda0d5851f74a423c83433d9674965e184642063 Mon Sep 17 00:00:00 2001 From: Jon Hagg Date: Mon, 26 Jul 2021 11:46:53 -0700 Subject: [PATCH 2/3] refactor: define scenario_id instance variable --- powersimdata/data_access/launcher.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py index f34646f22..09fec1f12 100644 --- a/powersimdata/data_access/launcher.py +++ b/powersimdata/data_access/launcher.py @@ -45,6 +45,7 @@ class Launcher: def __init__(self, scenario): self.scenario = scenario + self.scenario_id = scenario.scenario_id def _launch(self, threads=None, solver=None, extract_data=True): """Launches simulation on target environment @@ -102,7 +103,7 @@ def _run_script(self, script, extra_args=None): "python3", "-u", path_to_script, - self.scenario.scenario_id, + self.scenario_id, ] cmd_io_redirect = ["/dev/null 2>&1 &"] cmd = cmd_pythonpath + cmd_pythoncall + extra_args + cmd_io_redirect @@ -164,8 +165,7 @@ def _launch(self, threads=None, solver=None, extract_data=True): :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" keys which map to stdout, stderr, and the respective scenario attributes """ - scenario_id = self.scenario.scenario_id - url = f"{self.BASE_URL}/launch/{scenario_id}" + url = f"{self.BASE_URL}/launch/{self.scenario_id}" params = { "threads": threads, "solver": solver, @@ -184,8 +184,7 @@ def extract_simulation_output(self): :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" keys which map to stdout, stderr, and the respective scenario attributes """ - scenario_id = self.scenario.scenario_id - url = f"{self.BASE_URL}/extract/{scenario_id}" + url = f"{self.BASE_URL}/extract/{self.scenario_id}" resp = requests.post(url) return resp.json() @@ -195,8 +194,7 @@ def check_progress(self): :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" keys which map to stdout, stderr, and the respective scenario attributes """ - scenario_id = self.scenario.scenario_id - url = f"{self.BASE_URL}/status/{scenario_id}" + url = f"{self.BASE_URL}/status/{self.scenario_id}" resp = requests.get(url) return resp.json() @@ -220,9 +218,7 @@ def _launch(self, threads=None, solver=None, extract_data=True): """ from pyreisejl.utility import app - return app.launch_simulation( - self.scenario.scenario_id, threads, solver, extract_data - ) + return app.launch_simulation(self.scenario_id, threads, solver, extract_data) def extract_simulation_output(self): """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} @@ -232,7 +228,7 @@ def extract_simulation_output(self): """ from pyreisejl.utility import app - return app.extract_scenario(self.scenario.scenario_id) + return app.extract_scenario(self.scenario_id) def check_progress(self): """Get the status of an ongoing simulation, if possible From e83e677732c40e504026fa9677af96625e28d731 Mon Sep 17 00:00:00 2001 From: Jon Hagg Date: Mon, 26 Jul 2021 17:43:32 -0700 Subject: [PATCH 3/3] refactor: consolidate import statement --- powersimdata/data_access/launcher.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py index 09fec1f12..8598eea2e 100644 --- a/powersimdata/data_access/launcher.py +++ b/powersimdata/data_access/launcher.py @@ -1,3 +1,4 @@ +import importlib import posixpath import sys @@ -202,6 +203,7 @@ def check_progress(self): class NativeLauncher(Launcher): def __init__(self, scenario): sys.path.append(server_setup.ENGINE_DIR) + self.app = importlib.import_module("pyreisejl.utility.app") super().__init__(scenario) @@ -216,9 +218,9 @@ def _launch(self, threads=None, solver=None, extract_data=True): :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" keys which map to stdout, stderr, and the respective scenario attributes """ - from pyreisejl.utility import app - - return app.launch_simulation(self.scenario_id, threads, solver, extract_data) + return self.app.launch_simulation( + self.scenario_id, threads, solver, extract_data + ) def extract_simulation_output(self): """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} @@ -226,9 +228,7 @@ def extract_simulation_output(self): :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" keys which map to stdout, stderr, and the respective scenario attributes """ - from pyreisejl.utility import app - - return app.extract_scenario(self.scenario_id) + return self.app.extract_scenario(self.scenario_id) def check_progress(self): """Get the status of an ongoing simulation, if possible @@ -236,6 +236,4 @@ def check_progress(self): :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status" keys which map to stdout, stderr, and the respective scenario attributes """ - from pyreisejl.utility import app - - return app.check_progress() + return self.app.check_progress()