Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Define glpk launcher and expose command line and api option #112

Merged
merged 6 commits into from
Feb 23, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,14 @@ ENV PATH="$PATH:/usr/share/julia-1.5.3/bin" \
WORKDIR /app
COPY . .

RUN julia -e 'using Pkg; Pkg.activate("."); Pkg.instantiate(); Pkg.add("Gurobi"); import Gurobi; using REISE' && \
RUN julia -e 'using Pkg; \
Pkg.activate("."); \
Pkg.instantiate(); \
Pkg.add("Gurobi"); \
import Gurobi; \
Pkg.add("GLPK"); \
import GLPK; \
using REISE' && \
pip install -r requirements.txt


Expand Down
6 changes: 5 additions & 1 deletion pyreisejl/utility/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
Example request:

curl -XPOST http://localhost:5000/launch/1234
curl -XPOST http://localhost:5000/launch/1234?threads=42
curl -XPOST http://localhost:5000/launch/1234?threads=4&solver=glpk
curl http://localhost:5000/status/1234
"""

Expand All @@ -30,10 +30,14 @@ def get_script_path():
def launch_simulation(scenario_id):
cmd_call = ["python3", "-u", get_script_path(), str(scenario_id), "--extract-data"]
threads = request.args.get("threads", None)
solver = request.args.get("solver", None)

if threads is not None:
cmd_call.extend(["--threads", str(threads)])

if solver is not None:
cmd_call.extend(["--solver", solver])

proc = Popen(cmd_call, stdout=PIPE, stderr=PIPE, start_new_session=True)
entry = SimulationState(scenario_id, proc)
state.add(entry)
Expand Down
116 changes: 2 additions & 114 deletions pyreisejl/utility/call.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,14 @@
import os
from time import time

import pandas as pd

from pyreisejl.utility import const, parser
from pyreisejl.utility.extract_data import extract_scenario
from pyreisejl.utility.helpers import (
InvalidDateArgument,
InvalidInterval,
WrongNumberOfArguments,
extract_date_limits,
get_scenario,
insert_in_file,
sec2hms,
validate_time_format,
validate_time_range,
)
from pyreisejl.utility.launchers import get_launcher


def _record_scenario(scenario_id, runtime):
Expand All @@ -34,111 +27,6 @@ def _record_scenario(scenario_id, runtime):
)


class Launcher:
"""Parent class for solver-specific scenario launchers.

:param str start_date: start date of simulation as 'YYYY-MM-DD HH:MM:SS',
where HH, MM, and SS are optional.
:param str end_date: end date of simulation as 'YYYY-MM-DD HH:MM:SS',
where HH, MM, and SS are optional.
:param int interval: length of each interval in hours
:param str input_dir: directory with input data
:raises InvalidDateArgument: if start_date is posterior to end_date
:raises InvalidInterval: if the interval doesn't evently divide the given date range
"""

def __init__(self, start_date, end_date, interval, input_dir):
"""Constructor."""
# extract time limits from 'demand.csv'
with open(os.path.join(input_dir, "demand.csv")) as profile:
min_ts, max_ts, freq = extract_date_limits(profile)

dates = pd.date_range(start=min_ts, end=max_ts, freq=freq)

start_ts = validate_time_format(start_date)
end_ts = validate_time_format(end_date, end_date=True)

# make sure the dates are within the time frame we have data for
validate_time_range(start_ts, min_ts, max_ts)
validate_time_range(end_ts, min_ts, max_ts)

if start_ts > end_ts:
raise InvalidDateArgument(
f"The start date ({start_ts}) cannot be after the end date ({end_ts})."
)

# Julia starts at 1
start_index = dates.get_loc(start_ts) + 1
end_index = dates.get_loc(end_ts) + 1

# Calculate number of intervals
ts_range = end_index - start_index + 1
if ts_range % interval > 0:
raise InvalidInterval(
"This interval does not evenly divide the given date range."
)
self.start_index = start_index
self.interval = interval
self.n_interval = int(ts_range / interval)
self.input_dir = input_dir
print("Validation complete!")

def _print_settings(self):
print("Launching scenario with parameters:")
print(
{
"interval": self.interval,
"n_interval": self.n_interval,
"start_index": self.start_index,
"input_dir": self.input_dir,
"execute_dir": self.execute_dir,
"threads": self.threads,
}
)

def launch_scenario(self):
# This should be defined in sub-classes
raise NotImplementedError


class GurobiLauncher(Launcher):
def launch_scenario(self, execute_dir=None, threads=None, solver_kwargs=None):
"""Launches the scenario.

:param None/str execute_dir: directory for execute data. None defaults to an
execute folder that will be created in the input directory
:param None/int threads: number of threads to use.
:param None/dict solver_kwargs: keyword arguments to pass to solver (if any).
:return: (*int*) runtime of scenario in seconds
"""
self.execute_dir = execute_dir
self.threads = threads
self._print_settings()
# Import these within function because there is a lengthy compilation step
from julia.api import Julia

Julia(compiled_modules=False)
from julia import Gurobi # noqa: F401
from julia import REISE

start = time()
REISE.run_scenario_gurobi(
interval=self.interval,
n_interval=self.n_interval,
start_index=self.start_index,
inputfolder=self.input_dir,
outputfolder=self.execute_dir,
threads=self.threads,
)
end = time()

runtime = round(end - start)
hours, minutes, seconds = sec2hms(runtime)
print(f"Run time: {hours}:{minutes:02d}:{seconds:02d}")

return runtime


def main(args):
# Get scenario info if using PowerSimData
if args.scenario_id:
Expand All @@ -162,7 +50,7 @@ def main(args):
)
raise WrongNumberOfArguments(err_str)

launcher = GurobiLauncher(
launcher = get_launcher(args.solver)(
args.start_date,
args.end_date,
args.interval,
Expand Down
172 changes: 172 additions & 0 deletions pyreisejl/utility/launchers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
import os
from time import time

import pandas as pd

from pyreisejl.utility.helpers import (
InvalidDateArgument,
InvalidInterval,
extract_date_limits,
sec2hms,
validate_time_format,
validate_time_range,
)


class Launcher:
"""Parent class for solver-specific scenario launchers.

:param str start_date: start date of simulation as 'YYYY-MM-DD HH:MM:SS',
where HH, MM, and SS are optional.
:param str end_date: end date of simulation as 'YYYY-MM-DD HH:MM:SS',
where HH, MM, and SS are optional.
:param int interval: length of each interval in hours
:param str input_dir: directory with input data
:raises InvalidDateArgument: if start_date is posterior to end_date
:raises InvalidInterval: if the interval doesn't evently divide the given date range
"""

def __init__(self, start_date, end_date, interval, input_dir):
"""Constructor."""
# extract time limits from 'demand.csv'
with open(os.path.join(input_dir, "demand.csv")) as profile:
min_ts, max_ts, freq = extract_date_limits(profile)

dates = pd.date_range(start=min_ts, end=max_ts, freq=freq)

start_ts = validate_time_format(start_date)
end_ts = validate_time_format(end_date, end_date=True)

# make sure the dates are within the time frame we have data for
validate_time_range(start_ts, min_ts, max_ts)
validate_time_range(end_ts, min_ts, max_ts)

if start_ts > end_ts:
raise InvalidDateArgument(
f"The start date ({start_ts}) cannot be after the end date ({end_ts})."
)

# Julia starts at 1
start_index = dates.get_loc(start_ts) + 1
end_index = dates.get_loc(end_ts) + 1

# Calculate number of intervals
ts_range = end_index - start_index + 1
if ts_range % interval > 0:
raise InvalidInterval(
"This interval does not evenly divide the given date range."
)
self.start_index = start_index
self.interval = interval
self.n_interval = int(ts_range / interval)
self.input_dir = input_dir
print("Validation complete!")

def _print_settings(self):
print("Launching scenario with parameters:")
print(
{
"interval": self.interval,
"n_interval": self.n_interval,
"start_index": self.start_index,
"input_dir": self.input_dir,
"execute_dir": self.execute_dir,
"threads": self.threads,
}
)

def launch_scenario(self):
# This should be defined in sub-classes
raise NotImplementedError


class GLPKLauncher(Launcher):
def launch_scenario(self, execute_dir=None, threads=None, solver_kwargs=None):
"""Launches the scenario.

:param None/str execute_dir: directory for execute data. None defaults to an
execute folder that will be created in the input directory
:param None/int threads: number of threads to use.
:param None/dict solver_kwargs: keyword arguments to pass to solver (if any).
:return: (*int*) runtime of scenario in seconds
"""
self.execute_dir = execute_dir
self.threads = threads
self._print_settings()
print("INFO: threads not supported by GLPK, ignoring")

from julia.api import Julia

Julia(compiled_modules=False)
from julia import GLPK # noqa: F401
from julia import REISE

start = time()
REISE.run_scenario(
interval=self.interval,
n_interval=self.n_interval,
start_index=self.start_index,
inputfolder=self.input_dir,
outputfolder=self.execute_dir,
optimizer_factory=GLPK.Optimizer,
)
end = time()

runtime = round(end - start)
hours, minutes, seconds = sec2hms(runtime)
print(f"Run time: {hours}:{minutes:02d}:{seconds:02d}")

return runtime
Comment on lines +115 to +119
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we want to maximize our DRY, we could define a round_and_print_runtime method of the Launcher class and then call return self.round_and_print_runtime(start, end), but it's not really necessary.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I wanted to reuse the code here but since it's interleaved it felt like the result would be harder to read. If we end up adding more solvers I would want to do something about it.



class GurobiLauncher(Launcher):
def launch_scenario(self, execute_dir=None, threads=None, solver_kwargs=None):
"""Launches the scenario.

:param None/str execute_dir: directory for execute data. None defaults to an
execute folder that will be created in the input directory
:param None/int threads: number of threads to use.
:param None/dict solver_kwargs: keyword arguments to pass to solver (if any).
:return: (*int*) runtime of scenario in seconds
"""
self.execute_dir = execute_dir
self.threads = threads
self._print_settings()
# Import these within function because there is a lengthy compilation step
from julia.api import Julia

Julia(compiled_modules=False)
from julia import Gurobi # noqa: F401
from julia import REISE

start = time()
REISE.run_scenario_gurobi(
interval=self.interval,
n_interval=self.n_interval,
start_index=self.start_index,
inputfolder=self.input_dir,
outputfolder=self.execute_dir,
threads=self.threads,
)
end = time()

runtime = round(end - start)
hours, minutes, seconds = sec2hms(runtime)
print(f"Run time: {hours}:{minutes:02d}:{seconds:02d}")

return runtime


_launch_map = {"gurobi": GurobiLauncher, "glpk": GLPKLauncher}


def get_available_solvers():
return list(_launch_map.keys())


def get_launcher(solver):
if solver is None:
return GurobiLauncher
if solver.lower() not in _launch_map.keys():
raise ValueError("Invalid solver")
return _launch_map[solver]
9 changes: 9 additions & 0 deletions pyreisejl/utility/parser.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import argparse

from pyreisejl.utility.launchers import get_available_solvers


def parse_call_args():
parser = argparse.ArgumentParser(description="Run REISE.jl simulation.")
Expand Down Expand Up @@ -74,6 +76,13 @@ def parse_call_args():
"This flag is only used if the extract-data flag is set.",
)

solvers = ",".join(get_available_solvers())
parser.add_argument(
"--solver",
help="Specify the solver to run the optimization. Will default to gurobi. "
f"Current solvers available are {solvers}.",
)

# For backwards compatability with PowerSimData
parser.add_argument(
"scenario_id",
Expand Down