From e439bd35d5c5971890fdcbfac359b63db19c5c88 Mon Sep 17 00:00:00 2001
From: kasparm <10716532+kasparm@users.noreply.github.com>
Date: Fri, 31 Aug 2018 16:34:42 -0700
Subject: [PATCH 1/9] Added function to call scenario in parallel

---
 prereise/call/__init__.py |  3 ++-
 prereise/call/call.py     | 54 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/prereise/call/__init__.py b/prereise/call/__init__.py
index 0d27496da..633731925 100644
--- a/prereise/call/__init__.py
+++ b/prereise/call/__init__.py
@@ -1 +1,2 @@
-from .call import launch_scenario_performance
\ No newline at end of file
+from .call import launch_scenario_performance
+from .call import launch_scenario_performance_parallel
diff --git a/prereise/call/call.py b/prereise/call/call.py
index 7f1700ff2..0e311058e 100644
--- a/prereise/call/call.py
+++ b/prereise/call/call.py
@@ -2,12 +2,13 @@
 import pandas as pd
 import time
 import os
+from multiprocessing import Process
 
 import matlab.engine
-eng = matlab.engine.start_matlab()
 
 
 def launch_scenario_performance(scenario_name):
+
     # Load scenario list
     top_dirname = os.path.dirname(__file__)
     top_dirname = os.path.join(top_dirname, '../')
@@ -26,6 +27,8 @@ def launch_scenario_performance(scenario_name):
         os.mkdir(scenario.output_data_location.values[0])
     # Load path definition in matlab
     # TODO: This file need to be generated or should exist
+    # Start matlab engine
+    eng = matlab.engine.start_matlab()
     eng.run(top_dirname + 'add_path',nargout=0)
     eng.addpath(scenario.folder_location.values[0])
     eng.addpath(scenario.input_data_location.values[0])
@@ -39,6 +42,55 @@ def launch_scenario_performance(scenario_name):
     eng.rmpath(scenario.folder_location.values[0])
     eng.quit()
 
+def launch_scenario_performance_parallel(scenario_name,n_pcalls):
+    # Load scenario list
+
+    scenario_dirname = '/home/EGM/'
+    scenario_list = pd.read_csv(scenario_dirname + 'ScenarioList.csv')
+    
+    # Get parameters related to scenario
+    scenario = scenario_list[scenario_list.name == scenario_name]
+
+    # Catch if name not found
+    if scenario.shape[0] == 0:
+        print('No scenario with name ' + scenario_name)
+        return
+    # Create save data folder if does not exist
+    if not os.path.exists(scenario.output_data_location.values[0]):
+        os.mkdir(scenario.output_data_location.values[0])
+    # Load path definition in matlab
+    # TODO: This file need to be generated or should exist
+    # Start matlab engine
+    # Split the index into n_pcall parts
+    pcall_list = np.array_split(range(scenario.start_index.values[0],scenario.end_index.values[0]+1),n_pcalls)
+    proc = []
+    for i in pcall_list:      
+        p = Process(target = matlab_call, args=(scenario, int(i[0]), int(i[-1]),))
+        p.start()
+        proc.append(p)
+    for p in proc:
+        p.join()
+    
+def matlab_call(scenario, i_start, i_end):
+    top_dirname = os.path.dirname(__file__)
+    top_dirname = os.path.join(top_dirname, '../')
+    
+    eng = matlab.engine.start_matlab()
+    eng.run(top_dirname + 'add_path',nargout=0)
+    eng.addpath(scenario.folder_location.values[0])
+    eng.addpath(scenario.input_data_location.values[0])
+    print('output location')
+    print(scenario.output_data_location.values[0])
+    eng.workspace['output_data_location'] = scenario.output_data_location.values[0]
+    eng.workspace['start_index'] = i_start
+    eng.workspace['end_index'] = i_end
+    # Run scenario
+    eng.run(scenario.name.values[0],nargout=0)
+
+    eng.rmpath(scenario.input_data_location.values[0])
+    eng.rmpath(scenario.folder_location.values[0])
+    eng.quit()
+
 if __name__ == "__main__":
     import sys
     launch_scenario_performance(sys.argv[1])

From 0561c40ba1327801a3a700ff02488369047b8881 Mon Sep 17 00:00:00 2001
From: kasparm <10716532+kasparm@users.noreply.github.com>
Date: Mon, 1 Oct 2018 14:13:40 -0700
Subject: [PATCH 2/9] Cleaned up launch_scenario_performance to one function
 that is able to run in parallel.

---
 prereise/call/call.py           | 105 ++++++++++++++++----------------
 prereise/call/test/test_call.py |   7 ++-
 2 files changed, 55 insertions(+), 57 deletions(-)

diff --git a/prereise/call/call.py b/prereise/call/call.py
index 0e311058e..ec86ef045 100644
--- a/prereise/call/call.py
+++ b/prereise/call/call.py
@@ -1,20 +1,26 @@
-import numpy as np
-import pandas as pd
-import time
+import datetime
 import os
 from multiprocessing import Process
+from timeit import default_timer as timer
 
 import matlab.engine
+import numpy as np
+import pandas as pd
 
+# global variables
+scenario_dirname = '/home/EGM/'
+scenario_list = pd.read_csv(scenario_dirname + 'ScenarioList.csv')
 
-def launch_scenario_performance(scenario_name):
 
-    # Load scenario list
-    top_dirname = os.path.dirname(__file__)
-    top_dirname = os.path.join(top_dirname, '../')
-    scenario_dirname = '/home/EGM/'
-    scenario_list = pd.read_csv(scenario_dirname + 'ScenarioList.csv')
-    
+def launch_scenario_performance(scenario_name, n_pcalls=1):
+    """
+    This function launches the scenario.
+    The scenario is launched in parallel if n_pcalls > 1.
+    The function calls scenario_matlab_call in n_pcalls parallel calls.
+    :param scenario_name: name of the scenario.
+    :param n_pcalls: Number of parallel runs.
+    """
+
     # Get parameters related to scenario
     scenario = scenario_list[scenario_list.name == scenario_name]
 
@@ -25,72 +31,63 @@ def launch_scenario_performance(scenario_name):
     # Create save data folder if does not exist
     if not os.path.exists(scenario.output_data_location.values[0]):
         os.mkdir(scenario.output_data_location.values[0])
-    # Load path definition in matlab
-    # TODO: This file need to be generated or should exist
-    # Start matlab engine
-    eng = matlab.engine.start_matlab()
-    eng.run(top_dirname + 'add_path',nargout=0)
-    eng.addpath(scenario.folder_location.values[0])
-    eng.addpath(scenario.input_data_location.values[0])
-    eng.workspace['output_data_location'] = scenario.output_data_location.values[0]
-    eng.workspace['start_index'] = int(scenario.start_index.values[0])
-    eng.workspace['end_index'] = int(scenario.end_index.values[0])
-    # Run scenario
-    eng.run(scenario_name,nargout=0)
-
-    eng.rmpath(scenario.input_data_location.values[0])
-    eng.rmpath(scenario.folder_location.values[0])
-    eng.quit()
 
-def launch_scenario_performance_parallel(scenario_name,n_pcalls):
-    # Load scenario list
+    i_start = int(scenario.start_index.values[0])
+    i_end = int(scenario.end_index.values[0])
 
-    scenario_dirname = '/home/EGM/'
-    scenario_list = pd.read_csv(scenario_dirname + 'ScenarioList.csv')
-    
-    # Get parameters related to scenario
-    scenario = scenario_list[scenario_list.name == scenario_name]
+    if i_start < 1:
+        os.error('i_start has to be greater than 1')
+    if i_start > i_end:
+        os.error('i_end larger than i_start')
+    if n_pcalls > (i_end-i_start + 1):
+        os.error('n_pcalls is larger than the number of intervals')
 
-    # Catch if name not found
-    if scenario.shape[0] == 0:
-        print('No scenario with name ' + scenario_name)
-        return
-    # Create save data folder if does not exist
-    if not os.path.exists(scenario.output_data_location.values[0]):
-        os.mkdir(scenario.output_data_location.values[0])
-    # Load path definition in matlab
-    # TODO: This file need to be generated or should exist
-    # Start matlab engine
     # Split the index into n_pcall parts
-    pcall_list = np.array_split(range(scenario.start_index.values[0],scenario.end_index.values[0]+1),n_pcalls)
+    pcall_list = np.array_split(range(i_start, i_end+1), n_pcalls)
     proc = []
-    for i in pcall_list:      
-        p = Process(target = matlab_call, args=(scenario, int(i[0]), int(i[-1]),))
+    start = timer()
+    for i in pcall_list:
+        p = Process(target=scenario_matlab_call, args=(
+            scenario, int(i[0]), int(i[-1]),))
         p.start()
         proc.append(p)
     for p in proc:
         p.join()
-    
-def matlab_call(scenario, i_start, i_end):
+    end = timer()
+    print('Run time: ' + str(datetime.timedelta(seconds=(end-start))))
+
+
+def scenario_matlab_call(scenario, i_start, i_end):
+    """
+    It reads the scenario list file that contains all the information
+    related to the scenario. The function starts a MATLAB engine,
+    runs the add_path file to load MATPOWER and GUROBI.
+    It loads the data path and runs the scenario.
+    :param scenario: The scenario pandas data frame to be launched.
+    :param i_start: Start index.
+    :param i_end: End index.
+    """
+    # Location of add_path file
     top_dirname = os.path.dirname(__file__)
     top_dirname = os.path.join(top_dirname, '../')
-    
+
     eng = matlab.engine.start_matlab()
-    eng.run(top_dirname + 'add_path',nargout=0)
+    # Load path definition in MATLAB (MATPOWER and GUROBI)
+    eng.run(top_dirname + 'add_path', nargout=0)
     eng.addpath(scenario.folder_location.values[0])
     eng.addpath(scenario.input_data_location.values[0])
-    print('output location')
-    print(scenario.output_data_location.values[0])
-    eng.workspace['output_data_location'] = scenario.output_data_location.values[0]
+    eng.workspace['output_data_location'] = \
+        scenario.output_data_location.values[0]
     eng.workspace['start_index'] = i_start
     eng.workspace['end_index'] = i_end
     # Run scenario
-    eng.run(scenario.name.values[0],nargout=0)
+    eng.run(scenario.name.values[0], nargout=0)
 
     eng.rmpath(scenario.input_data_location.values[0])
     eng.rmpath(scenario.folder_location.values[0])
     eng.quit()
 
+
 if __name__ == "__main__":
     import sys
     launch_scenario_performance(sys.argv[1])
diff --git a/prereise/call/test/test_call.py b/prereise/call/test/test_call.py
index f310af636..b4833f736 100644
--- a/prereise/call/test/test_call.py
+++ b/prereise/call/test/test_call.py
@@ -1,7 +1,8 @@
+import sys
 import unittest
 
-import sys
-sys.path.append("..")
 import call
 
-call.launch_scenario_performance('texas_scenario')
\ No newline at end of file
+sys.path.append("..")
+
+call.launch_scenario_performance('western_scenarioUnitTest01', 16)

From 01934096bf5d56cd198aae22156d6245e004bc11 Mon Sep 17 00:00:00 2001
From: kasparm <10716532+kasparm@users.noreply.github.com>
Date: Tue, 2 Oct 2018 10:41:39 -0700
Subject: [PATCH 3/9] Added missing import and changed order of import in test

---
 prereise/call/__init__.py       | 1 -
 prereise/call/test/test_call.py | 5 +++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/prereise/call/__init__.py b/prereise/call/__init__.py
index 633731925..358a7902c 100644
--- a/prereise/call/__init__.py
+++ b/prereise/call/__init__.py
@@ -1,2 +1 @@
 from .call import launch_scenario_performance
-from .call import launch_scenario_performance_parallel
diff --git a/prereise/call/test/test_call.py b/prereise/call/test/test_call.py
index b4833f736..876f5acd2 100644
--- a/prereise/call/test/test_call.py
+++ b/prereise/call/test/test_call.py
@@ -1,8 +1,9 @@
 import sys
 import unittest
 
+sys.path.append("..")
+
 import call
 
-sys.path.append("..")
 
-call.launch_scenario_performance('western_scenarioUnitTest01', 16)
+call.launch_scenario_performance('western_scenarioUnitTest02', 16)

From 241d9f50d420d56c4e2b0d37bb91d01282c71470 Mon Sep 17 00:00:00 2001
From: kasparm <10716532+kasparm@users.noreply.github.com>
Date: Tue, 9 Oct 2018 16:02:49 -0700
Subject: [PATCH 4/9] style: Moved add_path into call

---
 prereise/{ => call}/add_path.m | 0
 prereise/call/call.py          | 3 +--
 setup.py                       | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)
 rename prereise/{ => call}/add_path.m (100%)

diff --git a/prereise/add_path.m b/prereise/call/add_path.m
similarity index 100%
rename from prereise/add_path.m
rename to prereise/call/add_path.m
diff --git a/prereise/call/call.py b/prereise/call/call.py
index ec86ef045..153143e42 100644
--- a/prereise/call/call.py
+++ b/prereise/call/call.py
@@ -69,11 +69,10 @@ def scenario_matlab_call(scenario, i_start, i_end):
     """
     # Location of add_path file
     top_dirname = os.path.dirname(__file__)
-    top_dirname = os.path.join(top_dirname, '../')
 
     eng = matlab.engine.start_matlab()
     # Load path definition in MATLAB (MATPOWER and GUROBI)
-    eng.run(top_dirname + 'add_path', nargout=0)
+    eng.run(top_dirname + '/add_path', nargout=0)
     eng.addpath(scenario.folder_location.values[0])
     eng.addpath(scenario.input_data_location.values[0])
     eng.workspace['output_data_location'] = \
diff --git a/setup.py b/setup.py
index 10721d4f6..1dfc78e7d 100644
--- a/setup.py
+++ b/setup.py
@@ -7,5 +7,5 @@
       author='Kaspar Mueller',
       author_email='kmueller@intven.com',
       packages=setuptools.find_packages(),
-      package_data={'prereise':['add_path.m']},
+      package_data={'prereise':['call/add_path.m']},
       zip_safe=False)

From a8ee2010bca5bb8638e913ce9eeeaaa93e219f44 Mon Sep 17 00:00:00 2001
From: kasparm <10716532+kasparm@users.noreply.github.com>
Date: Tue, 9 Oct 2018 16:03:51 -0700
Subject: [PATCH 5/9] style: test as a function

---
 prereise/call/test/test_call.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/prereise/call/test/test_call.py b/prereise/call/test/test_call.py
index 876f5acd2..7126c8325 100644
--- a/prereise/call/test/test_call.py
+++ b/prereise/call/test/test_call.py
@@ -1,9 +1,10 @@
 import sys
+import os
 import unittest
 
-sys.path.append("..")
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
 import call
 
-
-call.launch_scenario_performance('western_scenarioUnitTest02', 16)
+def test():
+    call.launch_scenario_performance('western_scenarioUnitTest02', 16)

From 199da3e705cb10e895a907adb2a24642288ddccd Mon Sep 17 00:00:00 2001
From: kasparm <10716532+kasparm@users.noreply.github.com>
Date: Tue, 9 Oct 2018 16:04:21 -0700
Subject: [PATCH 6/9] docs: README changes

---
 README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/README.md b/README.md
index ae83d48b0..e50ce36a0 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,7 @@ The convention is that we use the scenario name as the output folder name.
 Make sure your simulation m-file has the same name as the unique scenario name.
 
 ## Start the simulation.
+Simulation can only be launched on server. 
 After setting up the scenario, the simulation engine can be called.
 You launch the simulation the following way:
 ```python
@@ -26,6 +27,14 @@ import prereise
 
 prereise.launch_scenario_performance('scenario_name')
 ```
+### Test
+To test run:
+```python
+from prereise.call.test import test_call
+
+test_call.test()
+```
+
 ## Setup/Install
 This package requires Matlab, Gurobi, and Matpower. Make sure to put the paths
 from Gurobi and Matpower into the `add_path.m` file.

From ab92f0734c47d5720f2fd4398a2f126e8a645913 Mon Sep 17 00:00:00 2001
From: kasparm <10716532+kasparm@users.noreply.github.com>
Date: Tue, 9 Oct 2018 16:27:39 -0700
Subject: [PATCH 7/9] style: format file

---
 prereise/call/test/test_call.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/prereise/call/test/test_call.py b/prereise/call/test/test_call.py
index 7126c8325..50b5e7e39 100644
--- a/prereise/call/test/test_call.py
+++ b/prereise/call/test/test_call.py
@@ -1,10 +1,12 @@
-import sys
 import os
+import sys
 import unittest
 
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-
 import call
 
+sys.path.insert(0,
+                os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+
 def test():
     call.launch_scenario_performance('western_scenarioUnitTest02', 16)

From 0c66240c16c46be7da4c010b6461fbc18d9fc547 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ben=20Rouill=C3=A9=20d=27Orfeuil?= <brdo@intven.com>
Date: Wed, 17 Oct 2018 11:56:47 -0700
Subject: [PATCH 8/9] style: Format docstring

---
 prereise/call/call.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/prereise/call/call.py b/prereise/call/call.py
index 153143e42..bfd878ea0 100644
--- a/prereise/call/call.py
+++ b/prereise/call/call.py
@@ -17,6 +17,7 @@ def launch_scenario_performance(scenario_name, n_pcalls=1):
     This function launches the scenario.
     The scenario is launched in parallel if n_pcalls > 1.
     The function calls scenario_matlab_call in n_pcalls parallel calls.
+    
     :param scenario_name: name of the scenario.
     :param n_pcalls: Number of parallel runs.
     """
@@ -63,6 +64,7 @@ def scenario_matlab_call(scenario, i_start, i_end):
     related to the scenario. The function starts a MATLAB engine,
     runs the add_path file to load MATPOWER and GUROBI.
     It loads the data path and runs the scenario.
+    
     :param scenario: The scenario pandas data frame to be launched.
     :param i_start: Start index.
     :param i_end: End index.

From d259c8af9237ca18a6c3a28bf1419bda08ee39f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ben=20Rouill=C3=A9=20d=27Orfeuil?= <brdo@intven.com>
Date: Wed, 17 Oct 2018 11:58:06 -0700
Subject: [PATCH 9/9] chore: Create __all__

---
 prereise/call/__init__.py      | 2 +-
 prereise/call/test/__init__.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/prereise/call/__init__.py b/prereise/call/__init__.py
index 358a7902c..2fc624999 100644
--- a/prereise/call/__init__.py
+++ b/prereise/call/__init__.py
@@ -1 +1 @@
-from .call import launch_scenario_performance
+__all__ = ["call"]
diff --git a/prereise/call/test/__init__.py b/prereise/call/test/__init__.py
index e69de29bb..fb4c2737a 100644
--- a/prereise/call/test/__init__.py
+++ b/prereise/call/test/__init__.py
@@ -0,0 +1 @@
+__all__ = ["test_call"]