From e2cfbafcfb79707798f6f7a8e4dc8bd50d4266cb Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Jul 2020 19:40:46 +0000 Subject: [PATCH 01/15] Makes analysis workflow optional --- src/flownet/config_parser/_config_parser.py | 9 +-------- src/flownet/ert/_create_ert_setup.py | 2 +- src/flownet/templates/ahm_config.ert.jinja2 | 12 +++++++++++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/flownet/config_parser/_config_parser.py b/src/flownet/config_parser/_config_parser.py index 2f2fee5a2..878c323ad 100644 --- a/src/flownet/config_parser/_config_parser.py +++ b/src/flownet/config_parser/_config_parser.py @@ -119,6 +119,7 @@ def create_schema(_to_abs_path) -> Dict: "yamlobs": {MK.Type: types.String, MK.Required: False}, "analysis": { MK.Type: types.NamedDict, + MK.Required: False, MK.Content: { "metric": {MK.Type: types.String, MK.Required: False}, "quantity": {MK.Type: types.String, MK.Required: False}, @@ -421,14 +422,6 @@ def create_schema(_to_abs_path) -> Dict: / "static", "realizations": {"max_runtime": 300, "required_success_percent": 20}, "simulator": {"name": "flow"}, - "yamlobs": "./observations.yamlobs", - "analysis": { - "metric": "[RMSE]", - "quantity": "[WOPR:BR-P-]", - "start": "2001-04-01", - "end": "2006-01-01", - "outfile": "analysis_metrics_iteration", - }, }, "model_parameters": { "permeability": {"loguniform": True}, diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index bb0688c1e..5f0ca0376 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -125,7 +125,7 @@ def create_ert_setup( # pylint: disable=too-many-arguments with open(output_folder / "network.pickled", "wb") as fh: pickle.dump(network, fh) - # Pickle schdule + # Pickle schedule with open(output_folder / "schedule.pickled", "wb") as fh: pickle.dump(schedule, fh) diff --git a/src/flownet/templates/ahm_config.ert.jinja2 b/src/flownet/templates/ahm_config.ert.jinja2 index 06b393a70..177ad530d 100644 --- a/src/flownet/templates/ahm_config.ert.jinja2 +++ b/src/flownet/templates/ahm_config.ert.jinja2 @@ -3,15 +3,19 @@ INSTALL_JOB DELETE_IN_CURRENT_ITERATION ./DELETE_IN_CURRENT_ITERATION REFCASE /SYNTHETIC_REFCASE +{%- if ert_config.analysis: %} +DEFINE ../brugge_parameters.yml +{%- endif %} DEFINE {{ ert_config.runpath }} DEFINE {{ ert_config.eclbase }} DEFINE ./parameters.json -DEFINE {{ ert_config.yamlobs }} +{%- if ert_config.analysis: %} DEFINE {{ ert_config.analysis.metric }} DEFINE {{ ert_config.analysis.quantity }} DEFINE {{ ert_config.analysis.start }} DEFINE {{ ert_config.analysis.end }} DEFINE {{ ert_config.analysis.outfile }} +{%- endif %} INSTALL_JOB CREATE_FLOWNET_MODEL ./CREATE_FLOWNET_MODEL {%- if not debug: %} @@ -19,11 +23,17 @@ INSTALL_JOB DELETE_IN_CURRENT_ITERATION ./DELETE_IN_CURRENT_ITERATION {%- endif %} LOAD_WORKFLOW_JOB ./SAVE_ITERATION_PARAMETERS_WORKFLOW_JOB +{%- if ert_config.analysis: %} LOAD_WORKFLOW_JOB ./SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB +{%- endif %} LOAD_WORKFLOW SAVE_ITERATION_PARAMETERS_WORKFLOW +{%- if ert_config.analysis: %} LOAD_WORKFLOW SAVE_ITERATION_ANALYTICS_WORKFLOW +{%- endif %} HOOK_WORKFLOW SAVE_ITERATION_PARAMETERS_WORKFLOW PRE_SIMULATION +{%- if ert_config.analysis: %} HOOK_WORKFLOW SAVE_ITERATION_ANALYTICS_WORKFLOW POST_SIMULATION +{%- endif %} GEN_KW FLOWNET_PARAMETERS ./EMPTYFILE ./EMPTYFILE ./parameters.ertparam From 95de4cb02d7c4de3b99f70a14758dad8108fa98b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Jul 2020 19:42:27 +0000 Subject: [PATCH 02/15] Missing part from previous PR #81 --- src/flownet/ahm/_ahm_iteration_analytics.py | 5 ++--- src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW | 2 +- src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/flownet/ahm/_ahm_iteration_analytics.py b/src/flownet/ahm/_ahm_iteration_analytics.py index 2e258cda4..a0512f5db 100644 --- a/src/flownet/ahm/_ahm_iteration_analytics.py +++ b/src/flownet/ahm/_ahm_iteration_analytics.py @@ -336,13 +336,12 @@ def save_iteration_analytics(): Nothing """ - parser = argparse.ArgumentParser(prog=("Save iteration parameters to a file.")) - parser.add_argument("config", type=str, help="Path to the ERT config file.") + parser = argparse.ArgumentParser(prog=("Save iteration analytics to a file.")) + parser.add_argument("config", type=str, help="Path to the FlowNet config file.") parser.add_argument("runpath", type=str, help="Path to the ERT runpath.") parser.add_argument( "eclbase", type=str, help="Path to the simulation from runpath." ) - parser.add_argument("yamlobs", type=str, help="Path to the yaml observation file.") parser.add_argument( "start", type=str, help="Start date (YYYY-MM-DD) for accuracy analysis." ) diff --git a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW index 8ed1cb86b..8a8280ea1 100644 --- a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW +++ b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW @@ -1,2 +1,2 @@ -SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB +SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB diff --git a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB index db2ab2414..6b7b82cff 100644 --- a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB +++ b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB @@ -10,4 +10,3 @@ ARG_TYPE 4 STRING ARG_TYPE 5 STRING ARG_TYPE 6 STRING ARG_TYPE 7 STRING - From 2f6387ce4279e87a0da7c9aaeb026118ec246931 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 6 Jul 2020 12:38:16 +0000 Subject: [PATCH 03/15] Reduces number of inputs to analysis workflow --- src/flownet/ahm/_ahm_iteration_analytics.py | 54 +++++++++---------- .../static/SAVE_ITERATION_ANALYTICS_WORKFLOW | 3 +- .../SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB | 9 +--- src/flownet/templates/ahm_config.ert.jinja2 | 15 +----- 4 files changed, 31 insertions(+), 50 deletions(-) diff --git a/src/flownet/ahm/_ahm_iteration_analytics.py b/src/flownet/ahm/_ahm_iteration_analytics.py index a0512f5db..459e07d8b 100644 --- a/src/flownet/ahm/_ahm_iteration_analytics.py +++ b/src/flownet/ahm/_ahm_iteration_analytics.py @@ -342,41 +342,27 @@ def save_iteration_analytics(): parser.add_argument( "eclbase", type=str, help="Path to the simulation from runpath." ) - parser.add_argument( - "start", type=str, help="Start date (YYYY-MM-DD) for accuracy analysis." - ) - parser.add_argument( - "end", type=str, help="End date (YYYY-MM-DD) for accuracy analysis." - ) - parser.add_argument( - "quantity", - type=str, - help="List of names of quantities of interest for accuracy analysis.", - ) - parser.add_argument("metrics", type=str, help="List of names of accuracy metrics.") - parser.add_argument( - "outfile", - type=str, - help="Name of output file containing metrics over iterations.", - ) args = parser.parse_args() args.runpath = args.runpath.replace("%d", "*") print("Saving iteration analytics...", end=" ") + # Load FlowNet config file + config = parse_config(pathlib.Path(args.config)) + # Fix list inputs - metrics = list(args.metrics.replace("[", "").replace("]", "").split(",")) + metrics = list( + config.ert.analysis.metric.replace("[", "").replace("]", "").split(",") + ) # Load ensemble of FlowNet (df_sim, realizations_dict, iteration,) = make_dataframe_simulation_data( args.runpath, args.eclbase, - list(args.quantity.replace("[", "").replace("]", "").split(",")), + list(config.ert.analysis.quantity.replace("[", "").replace("]", "").split(",")), ) # Load reference simulation (OPM-Flow/Eclipse) - config = parse_config(pathlib.Path(args.config)) - field_data = FlowData( config.flownet.data_source.input_case, perforation_handling_strategy=config.flownet.perforation_handling_strategy, @@ -392,12 +378,19 @@ def save_iteration_analytics(): df_sim = df_sim[df_sim["DATE"].isin(df_obs["DATE"])] # Initiate dataframe with metrics - df_metrics = load_csv_file(args.outfile, ["quantity", "iteration"] + metrics) + df_metrics = load_csv_file( + config.ert.analysis.outfile, ["quantity", "iteration"] + metrics + ) - for str_key in list(args.quantity.replace("[", "").replace("]", "").split(",")): + for str_key in list( + config.ert.analysis.quantity.replace("[", "").replace("]", "").split(",") + ): # Prepare data from reference simulation tmp_data = filter_dataframe( - df_obs, "DATE", np.datetime64(args.start), np.datetime64(args.end) + df_obs, + "DATE", + np.datetime64(config.ert.analysis.start), + np.datetime64(config.ert.analysis.end), ) truth_data = pd.DataFrame() @@ -406,7 +399,6 @@ def save_iteration_analytics(): truth_data[str_key[:5] + well] = tmp_data[tmp_data["WELL_NAME"] == well][ str_key[:4] ].values - truth_data.to_csv("truth_data.csv", index=False) obs_opm = prepare_opm_reference_data( truth_data, str_key, len(realizations_dict) @@ -417,7 +409,10 @@ def save_iteration_analytics(): ens_flownet.append( prepare_flownet_data( filter_dataframe( - df_sim, "DATE", np.datetime64(args.start), np.datetime64(args.end) + df_sim, + "DATE", + np.datetime64(config.ert.analysis.start), + np.datetime64(config.ert.analysis.end), ), str_key, len(realizations_dict), @@ -425,7 +420,10 @@ def save_iteration_analytics(): ) filter_dataframe( - df_sim, "DATE", np.datetime64(args.start), np.datetime64(args.end) + df_sim, + "DATE", + np.datetime64(config.ert.analysis.start), + np.datetime64(config.ert.analysis.end), ).to_csv("ens_flownet_data.csv", index=False) # Normalizing data @@ -441,6 +439,6 @@ def save_iteration_analytics(): save_plots_metrics(df_metrics, metrics, str_key) # Saving accuracy metrics to CSV file - df_metrics.to_csv(args.outfile + ".csv", index=False) + df_metrics.to_csv(config.ert.analysis.outfile + ".csv", index=False) print("[Done]") diff --git a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW index 8a8280ea1..36680318d 100644 --- a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW +++ b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW @@ -1,2 +1 @@ -SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB - +SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB \ No newline at end of file diff --git a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB index 6b7b82cff..0744928ed 100644 --- a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB +++ b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB @@ -1,12 +1,7 @@ INTERNAL FALSE EXECUTABLE flownet_save_iteration_analytics -MIN_ARG 8 -MAX_ARG 8 +MIN_ARG 3 +MAX_ARG 3 ARG_TYPE 0 STRING ARG_TYPE 1 STRING ARG_TYPE 2 STRING -ARG_TYPE 3 STRING -ARG_TYPE 4 STRING -ARG_TYPE 5 STRING -ARG_TYPE 6 STRING -ARG_TYPE 7 STRING diff --git a/src/flownet/templates/ahm_config.ert.jinja2 b/src/flownet/templates/ahm_config.ert.jinja2 index 177ad530d..fbfa7d1aa 100644 --- a/src/flownet/templates/ahm_config.ert.jinja2 +++ b/src/flownet/templates/ahm_config.ert.jinja2 @@ -9,13 +9,6 @@ DEFINE ../brugge_parameters.yml DEFINE {{ ert_config.runpath }} DEFINE {{ ert_config.eclbase }} DEFINE ./parameters.json -{%- if ert_config.analysis: %} -DEFINE {{ ert_config.analysis.metric }} -DEFINE {{ ert_config.analysis.quantity }} -DEFINE {{ ert_config.analysis.start }} -DEFINE {{ ert_config.analysis.end }} -DEFINE {{ ert_config.analysis.outfile }} -{%- endif %} INSTALL_JOB CREATE_FLOWNET_MODEL ./CREATE_FLOWNET_MODEL {%- if not debug: %} @@ -23,15 +16,11 @@ INSTALL_JOB DELETE_IN_CURRENT_ITERATION ./DELETE_IN_CURRENT_ITERATION {%- endif %} LOAD_WORKFLOW_JOB ./SAVE_ITERATION_PARAMETERS_WORKFLOW_JOB -{%- if ert_config.analysis: %} -LOAD_WORKFLOW_JOB ./SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB -{%- endif %} LOAD_WORKFLOW SAVE_ITERATION_PARAMETERS_WORKFLOW -{%- if ert_config.analysis: %} -LOAD_WORKFLOW SAVE_ITERATION_ANALYTICS_WORKFLOW -{%- endif %} HOOK_WORKFLOW SAVE_ITERATION_PARAMETERS_WORKFLOW PRE_SIMULATION {%- if ert_config.analysis: %} +LOAD_WORKFLOW_JOB ./SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB +LOAD_WORKFLOW SAVE_ITERATION_ANALYTICS_WORKFLOW HOOK_WORKFLOW SAVE_ITERATION_ANALYTICS_WORKFLOW POST_SIMULATION {%- endif %} From 4941a83404891aa673b9d2c0dd0ba282bf67009b Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 09:22:24 +0000 Subject: [PATCH 04/15] Renders input to workflow in ERT config file --- src/flownet/ahm/_ahm_iteration_analytics.py | 62 ++++++++++--------- src/flownet/ahm/_assisted_history_matching.py | 8 +++ src/flownet/ahm/_run_ahm.py | 2 + src/flownet/ert/_create_ert_setup.py | 7 +++ .../static/SAVE_ITERATION_ANALYTICS_WORKFLOW | 2 +- .../SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB | 10 ++- src/flownet/templates/ahm_config.ert.jinja2 | 8 ++- 7 files changed, 66 insertions(+), 33 deletions(-) diff --git a/src/flownet/ahm/_ahm_iteration_analytics.py b/src/flownet/ahm/_ahm_iteration_analytics.py index 459e07d8b..c1ab66a07 100644 --- a/src/flownet/ahm/_ahm_iteration_analytics.py +++ b/src/flownet/ahm/_ahm_iteration_analytics.py @@ -337,35 +337,52 @@ def save_iteration_analytics(): """ parser = argparse.ArgumentParser(prog=("Save iteration analytics to a file.")) - parser.add_argument("config", type=str, help="Path to the FlowNet config file.") + parser.add_argument( + "reference_simulation", type=str, help="Path to the reference simulation case" + ) + parser.add_argument( + "perforation_strategy", type=str, help="Perforation handling strategy" + ) parser.add_argument("runpath", type=str, help="Path to the ERT runpath.") parser.add_argument( "eclbase", type=str, help="Path to the simulation from runpath." ) + parser.add_argument( + "start", type=str, help="Start date (YYYY-MM-DD) for accuracy analysis." + ) + parser.add_argument( + "end", type=str, help="End date (YYYY-MM-DD) for accuracy analysis." + ) + parser.add_argument( + "quantity", + type=str, + help="List of names of quantities of interest for accuracy analysis.", + ) + parser.add_argument("metrics", type=str, help="List of names of accuracy metrics.") + parser.add_argument( + "outfile", + type=str, + help="Name of output file containing metrics over iterations.", + ) args = parser.parse_args() args.runpath = args.runpath.replace("%d", "*") print("Saving iteration analytics...", end=" ") - # Load FlowNet config file - config = parse_config(pathlib.Path(args.config)) - # Fix list inputs - metrics = list( - config.ert.analysis.metric.replace("[", "").replace("]", "").split(",") - ) + metrics = list(args.metrics.replace("[", "").replace("]", "").split(",")) # Load ensemble of FlowNet (df_sim, realizations_dict, iteration,) = make_dataframe_simulation_data( args.runpath, args.eclbase, - list(config.ert.analysis.quantity.replace("[", "").replace("]", "").split(",")), + list(args.quantity.replace("[", "").replace("]", "").split(",")), ) # Load reference simulation (OPM-Flow/Eclipse) field_data = FlowData( - config.flownet.data_source.input_case, - perforation_handling_strategy=config.flownet.perforation_handling_strategy, + args.reference_simulation, + perforation_handling_strategy=args.perforation_strategy, ) df_obs: pd.DataFrame = field_data.production df_obs["DATE"] = df_obs["date"] @@ -378,19 +395,12 @@ def save_iteration_analytics(): df_sim = df_sim[df_sim["DATE"].isin(df_obs["DATE"])] # Initiate dataframe with metrics - df_metrics = load_csv_file( - config.ert.analysis.outfile, ["quantity", "iteration"] + metrics - ) + df_metrics = load_csv_file(args.outfile, ["quantity", "iteration"] + metrics) - for str_key in list( - config.ert.analysis.quantity.replace("[", "").replace("]", "").split(",") - ): + for str_key in list(args.quantity.replace("[", "").replace("]", "").split(",")): # Prepare data from reference simulation tmp_data = filter_dataframe( - df_obs, - "DATE", - np.datetime64(config.ert.analysis.start), - np.datetime64(config.ert.analysis.end), + df_obs, "DATE", np.datetime64(args.start), np.datetime64(args.end), ) truth_data = pd.DataFrame() @@ -409,10 +419,7 @@ def save_iteration_analytics(): ens_flownet.append( prepare_flownet_data( filter_dataframe( - df_sim, - "DATE", - np.datetime64(config.ert.analysis.start), - np.datetime64(config.ert.analysis.end), + df_sim, "DATE", np.datetime64(args.start), np.datetime64(args.end), ), str_key, len(realizations_dict), @@ -420,10 +427,7 @@ def save_iteration_analytics(): ) filter_dataframe( - df_sim, - "DATE", - np.datetime64(config.ert.analysis.start), - np.datetime64(config.ert.analysis.end), + df_sim, "DATE", np.datetime64(args.start), np.datetime64(args.end), ).to_csv("ens_flownet_data.csv", index=False) # Normalizing data @@ -439,6 +443,6 @@ def save_iteration_analytics(): save_plots_metrics(df_metrics, metrics, str_key) # Saving accuracy metrics to CSV file - df_metrics.to_csv(config.ert.analysis.outfile + ".csv", index=False) + df_metrics.to_csv(args.outfile + ".csv", index=False) print("[Done]") diff --git a/src/flownet/ahm/_assisted_history_matching.py b/src/flownet/ahm/_assisted_history_matching.py index baf837bf3..35b8742c7 100644 --- a/src/flownet/ahm/_assisted_history_matching.py +++ b/src/flownet/ahm/_assisted_history_matching.py @@ -39,6 +39,8 @@ def __init__( schedule: Schedule, parameters: List[Parameter], case_name: str, + perforation_strategy: str, + reference_simulation: str, ert_config: Dict, random_seed: Optional[int] = None, ): @@ -50,6 +52,8 @@ def __init__( schedule: Schedule instance parameters: List of Parameter objects case_name: Name of simulation case + perforation_strategy: String indicating perforation handling strategy + reference_simulation: String indicating path to reference simulation case ert_config: Dictionary containing information about queue (system, name, server and max_running) and realizations (num_realizations, required_success_percent and max_runtime) random_seed: Random seed to control reproducibility of FlowNet @@ -58,6 +62,8 @@ def __init__( self._network: NetworkModel = network self._schedule: Schedule = schedule self._parameters: List[Parameter] = parameters + self._perforation_strategy: str = perforation_strategy + self._reference_simulation: str = reference_simulation self._ert_config: dict = ert_config self._case_name: str = case_name self._random_seed: Optional[int] = random_seed @@ -82,6 +88,8 @@ def create_ert_setup(self, args: argparse.Namespace, training_set_fraction: floa args, self._network, self._schedule, + perforation_strategy=self._perforation_strategy, + reference_simulation=self._reference_simulation, ert_config=self._ert_config, parameters=self._parameters, random_seed=self._random_seed, diff --git a/src/flownet/ahm/_run_ahm.py b/src/flownet/ahm/_run_ahm.py index 0a769ac63..a8d57197d 100644 --- a/src/flownet/ahm/_run_ahm.py +++ b/src/flownet/ahm/_run_ahm.py @@ -388,6 +388,8 @@ def run_flownet_history_matching( schedule, parameters, case_name=config.name, + perforation_strategy=config.flownet.perforation_handling_strategy, + reference_simulation=config.flownet.data_source.input_case, ert_config=config.ert._asdict(), random_seed=config.flownet.random_seed, ) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 5f0ca0376..14b1785e3 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -91,6 +91,8 @@ def create_ert_setup( # pylint: disable=too-many-arguments args: argparse.Namespace, network, schedule: Schedule, + perforation_strategy: str, + reference_simulation: str, ert_config: dict, parameters=None, random_seed=None, @@ -114,6 +116,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments output_folder = pathlib.Path(args.output_folder) os.makedirs(output_folder, exist_ok=True) + # Derive absolute path to reference simulation case + reference_simulation = pathlib.Path(reference_simulation).resolve() + if prediction_setup: ert_config_file = output_folder / "pred_config.ert" template = _TEMPLATE_ENVIRONMENT.get_template("pred_config.ert.jinja2") @@ -141,6 +146,8 @@ def create_ert_setup( # pylint: disable=too-many-arguments "pickled_parameters": output_folder.resolve() / "parameters.pickled", "random_seed": random_seed, + "perforation_strategy": perforation_strategy, + "reference_simulation": reference_simulation, "ert_config": ert_config, "debug": args.debug if hasattr(args, "debug") else False, "pred_schedule_file": getattr( diff --git a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW index 36680318d..54646075a 100644 --- a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW +++ b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW @@ -1 +1 @@ -SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB \ No newline at end of file +SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB diff --git a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB index 0744928ed..c1114271f 100644 --- a/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB +++ b/src/flownet/static/SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB @@ -1,7 +1,13 @@ INTERNAL FALSE EXECUTABLE flownet_save_iteration_analytics -MIN_ARG 3 -MAX_ARG 3 +MIN_ARG 9 +MAX_ARG 9 ARG_TYPE 0 STRING ARG_TYPE 1 STRING ARG_TYPE 2 STRING +ARG_TYPE 3 STRING +ARG_TYPE 4 STRING +ARG_TYPE 5 STRING +ARG_TYPE 6 STRING +ARG_TYPE 7 STRING +ARG_TYPE 8 STRING diff --git a/src/flownet/templates/ahm_config.ert.jinja2 b/src/flownet/templates/ahm_config.ert.jinja2 index fbfa7d1aa..bcfba68bd 100644 --- a/src/flownet/templates/ahm_config.ert.jinja2 +++ b/src/flownet/templates/ahm_config.ert.jinja2 @@ -4,7 +4,13 @@ INSTALL_JOB DELETE_IN_CURRENT_ITERATION ./DELETE_IN_CURRENT_ITERATION REFCASE /SYNTHETIC_REFCASE {%- if ert_config.analysis: %} -DEFINE ../brugge_parameters.yml +DEFINE {{ ert_config.analysis.metric }} +DEFINE {{ ert_config.analysis.quantity }} +DEFINE {{ ert_config.analysis.start }} +DEFINE {{ ert_config.analysis.end }} +DEFINE {{ ert_config.analysis.outfile }} +DEFINE {{ perforation_strategy }} +DEFINE {{ reference_simulation }} {%- endif %} DEFINE {{ ert_config.runpath }} DEFINE {{ ert_config.eclbase }} From 007d82ea1b67f30f2f075075be7af44bbfbb4dbb Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 10:13:38 +0000 Subject: [PATCH 05/15] Removes unnecessary import --- src/flownet/ahm/_ahm_iteration_analytics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/flownet/ahm/_ahm_iteration_analytics.py b/src/flownet/ahm/_ahm_iteration_analytics.py index c1ab66a07..6e7d25c20 100644 --- a/src/flownet/ahm/_ahm_iteration_analytics.py +++ b/src/flownet/ahm/_ahm_iteration_analytics.py @@ -14,7 +14,6 @@ from ecl.summary import EclSum -from ..config_parser import parse_config from ..data import FlowData From b2ca21e14fa91e328556aa3b9717a77fac9a0461 Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 10:34:48 +0000 Subject: [PATCH 06/15] Makes new arguments optional to work with run_pred --- src/flownet/ert/_create_ert_setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 14b1785e3..1a8b20aec 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -91,9 +91,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments args: argparse.Namespace, network, schedule: Schedule, - perforation_strategy: str, - reference_simulation: str, ert_config: dict, + perforation_strategy: str = None, + reference_simulation: str = None, parameters=None, random_seed=None, training_set_fraction: float = 1, From 0abc1680036c807c1f57bfb2b0f8e495887bb8de Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 10:42:42 +0000 Subject: [PATCH 07/15] Fixes type of variable for path to reference simulation --- src/flownet/ert/_create_ert_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 1a8b20aec..2942e244b 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -93,7 +93,7 @@ def create_ert_setup( # pylint: disable=too-many-arguments schedule: Schedule, ert_config: dict, perforation_strategy: str = None, - reference_simulation: str = None, + reference_simulation: pathlib.Path = None, parameters=None, random_seed=None, training_set_fraction: float = 1, From 4f07d4eec584cc1ea541aa6f2346bc3d7d6f44a8 Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 10:51:41 +0000 Subject: [PATCH 08/15] Fixes path type variable in ahm --- src/flownet/ahm/_assisted_history_matching.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/flownet/ahm/_assisted_history_matching.py b/src/flownet/ahm/_assisted_history_matching.py index c4aec336b..66fa96cb9 100644 --- a/src/flownet/ahm/_assisted_history_matching.py +++ b/src/flownet/ahm/_assisted_history_matching.py @@ -40,7 +40,7 @@ def __init__( parameters: List[Parameter], case_name: str, perforation_strategy: str, - reference_simulation: str, + reference_simulation: pathlib.Path, ert_config: Dict, random_seed: Optional[int] = None, ): @@ -53,7 +53,7 @@ def __init__( parameters: List of Parameter objects case_name: Name of simulation case perforation_strategy: String indicating perforation handling strategy - reference_simulation: String indicating path to reference simulation case + reference_simulation: pathlib.Path indicating path to reference simulation case ert_config: Dictionary containing information about queue (system, name, server and max_running) and realizations (num_realizations, required_success_percent and max_runtime) random_seed: Random seed to control reproducibility of FlowNet @@ -63,7 +63,7 @@ def __init__( self._schedule: Schedule = schedule self._parameters: List[Parameter] = parameters self._perforation_strategy: str = perforation_strategy - self._reference_simulation: str = reference_simulation + self._reference_simulation: pathlib.Path = reference_simulation self._ert_config: dict = ert_config self._case_name: str = case_name self._random_seed: Optional[int] = random_seed From 9899bd3db07d6bbcb5458d25b337a71af5efeb18 Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 11:35:30 +0000 Subject: [PATCH 09/15] Revert back to path as string --- src/flownet/ert/_create_ert_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 2942e244b..1a8b20aec 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -93,7 +93,7 @@ def create_ert_setup( # pylint: disable=too-many-arguments schedule: Schedule, ert_config: dict, perforation_strategy: str = None, - reference_simulation: pathlib.Path = None, + reference_simulation: str = None, parameters=None, random_seed=None, training_set_fraction: float = 1, From a8df4ef3e0732e2c6242f41bfdb5a2d526f5079a Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 11:42:32 +0000 Subject: [PATCH 10/15] Revert path back to string in ahm --- src/flownet/ahm/_assisted_history_matching.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/flownet/ahm/_assisted_history_matching.py b/src/flownet/ahm/_assisted_history_matching.py index 66fa96cb9..c4aec336b 100644 --- a/src/flownet/ahm/_assisted_history_matching.py +++ b/src/flownet/ahm/_assisted_history_matching.py @@ -40,7 +40,7 @@ def __init__( parameters: List[Parameter], case_name: str, perforation_strategy: str, - reference_simulation: pathlib.Path, + reference_simulation: str, ert_config: Dict, random_seed: Optional[int] = None, ): @@ -53,7 +53,7 @@ def __init__( parameters: List of Parameter objects case_name: Name of simulation case perforation_strategy: String indicating perforation handling strategy - reference_simulation: pathlib.Path indicating path to reference simulation case + reference_simulation: String indicating path to reference simulation case ert_config: Dictionary containing information about queue (system, name, server and max_running) and realizations (num_realizations, required_success_percent and max_runtime) random_seed: Random seed to control reproducibility of FlowNet @@ -63,7 +63,7 @@ def __init__( self._schedule: Schedule = schedule self._parameters: List[Parameter] = parameters self._perforation_strategy: str = perforation_strategy - self._reference_simulation: pathlib.Path = reference_simulation + self._reference_simulation: str = reference_simulation self._ert_config: dict = ert_config self._case_name: str = case_name self._random_seed: Optional[int] = random_seed From 7dd83eb2c295e1c647d3af67ad194af125b1aa05 Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 11:49:02 +0000 Subject: [PATCH 11/15] Fixes derivation of absolute path --- src/flownet/ert/_create_ert_setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 1a8b20aec..6bca06c80 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -117,7 +117,8 @@ def create_ert_setup( # pylint: disable=too-many-arguments os.makedirs(output_folder, exist_ok=True) # Derive absolute path to reference simulation case - reference_simulation = pathlib.Path(reference_simulation).resolve() + if reference_simulation: + reference_simulation = pathlib.Path(reference_simulation).resolve() if prediction_setup: ert_config_file = output_folder / "pred_config.ert" From 1e5ee40ae48d94db824044f96409a1f9ecbdc111 Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 17:04:20 +0000 Subject: [PATCH 12/15] Fixes it for new configsuite version --- src/flownet/config_parser/_config_parser.py | 17 +++++------------ src/flownet/ert/_create_ert_setup.py | 4 ++-- src/flownet/templates/ahm_config.ert.jinja2 | 4 ++-- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/flownet/config_parser/_config_parser.py b/src/flownet/config_parser/_config_parser.py index bcdfc9f4f..545d3180d 100644 --- a/src/flownet/config_parser/_config_parser.py +++ b/src/flownet/config_parser/_config_parser.py @@ -129,19 +129,12 @@ def create_schema(_to_abs_path) -> Dict: }, "analysis": { MK.Type: types.NamedDict, - MK.Required: False, MK.Content: { - "metric": {MK.Type: types.String, MK.Default: "[RMSE]"}, - "quantity": { - MK.Type: types.String, - MK.Default: "[WOPR:BR-P-]", - }, - "start": {MK.Type: types.String, MK.Default: "2001-04-01"}, - "end": {MK.Type: types.String, MK.Default: "2006-01-01"}, - "outfile": { - MK.Type: types.String, - MK.Default: "analysis_metrics_iteration", - }, + "metric": {MK.Type: types.String, MK.AllowNone: True}, + "quantity": {MK.Type: types.String, MK.AllowNone: True,}, + "start": {MK.Type: types.String, MK.AllowNone: True}, + "end": {MK.Type: types.String, MK.AllowNone: True}, + "outfile": {MK.Type: types.String, MK.AllowNone: True,}, }, }, }, diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 6bca06c80..0acbe4b8d 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -92,10 +92,10 @@ def create_ert_setup( # pylint: disable=too-many-arguments network, schedule: Schedule, ert_config: dict, - perforation_strategy: str = None, - reference_simulation: str = None, parameters=None, random_seed=None, + perforation_strategy: str = None, + reference_simulation: str = None, training_set_fraction: float = 1, prediction_setup: bool = False, ): diff --git a/src/flownet/templates/ahm_config.ert.jinja2 b/src/flownet/templates/ahm_config.ert.jinja2 index bcfba68bd..18df90593 100644 --- a/src/flownet/templates/ahm_config.ert.jinja2 +++ b/src/flownet/templates/ahm_config.ert.jinja2 @@ -3,7 +3,7 @@ INSTALL_JOB DELETE_IN_CURRENT_ITERATION ./DELETE_IN_CURRENT_ITERATION REFCASE /SYNTHETIC_REFCASE -{%- if ert_config.analysis: %} +{%- if ert_config.analysis.metric: %} DEFINE {{ ert_config.analysis.metric }} DEFINE {{ ert_config.analysis.quantity }} DEFINE {{ ert_config.analysis.start }} @@ -24,7 +24,7 @@ INSTALL_JOB DELETE_IN_CURRENT_ITERATION ./DELETE_IN_CURRENT_ITERATION LOAD_WORKFLOW_JOB ./SAVE_ITERATION_PARAMETERS_WORKFLOW_JOB LOAD_WORKFLOW SAVE_ITERATION_PARAMETERS_WORKFLOW HOOK_WORKFLOW SAVE_ITERATION_PARAMETERS_WORKFLOW PRE_SIMULATION -{%- if ert_config.analysis: %} +{%- if ert_config.analysis.metric: %} LOAD_WORKFLOW_JOB ./SAVE_ITERATION_ANALYTICS_WORKFLOW_JOB LOAD_WORKFLOW SAVE_ITERATION_ANALYTICS_WORKFLOW HOOK_WORKFLOW SAVE_ITERATION_ANALYTICS_WORKFLOW POST_SIMULATION From 038583a99f80c3edaf485815e39e6dd1a87fd516 Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 17:16:26 +0000 Subject: [PATCH 13/15] Fixes for CI error on variable type --- src/flownet/ert/_create_ert_setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 0acbe4b8d..ab78c693d 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -118,7 +118,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments # Derive absolute path to reference simulation case if reference_simulation: - reference_simulation = pathlib.Path(reference_simulation).resolve() + path_ref_sim = pathlib.Path(reference_simulation).resolve() + else: + path_ref_sim = None if prediction_setup: ert_config_file = output_folder / "pred_config.ert" @@ -148,7 +150,7 @@ def create_ert_setup( # pylint: disable=too-many-arguments / "parameters.pickled", "random_seed": random_seed, "perforation_strategy": perforation_strategy, - "reference_simulation": reference_simulation, + "reference_simulation": path_ref_sim, "ert_config": ert_config, "debug": args.debug if hasattr(args, "debug") else False, "pred_schedule_file": getattr( From 98a7a42ac7c23c367349b6c449c32bc7257d72ac Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 17:27:22 +0000 Subject: [PATCH 14/15] Fixes for CI error on variable type (2) --- src/flownet/ert/_create_ert_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index ab78c693d..7edb89063 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -120,7 +120,7 @@ def create_ert_setup( # pylint: disable=too-many-arguments if reference_simulation: path_ref_sim = pathlib.Path(reference_simulation).resolve() else: - path_ref_sim = None + path_ref_sim = pathlib.Path('.').resolve() if prediction_setup: ert_config_file = output_folder / "pred_config.ert" From 659e9e9a60069cf5f9e4da1e588ac4b2e1583d99 Mon Sep 17 00:00:00 2001 From: Eduardo Barros Date: Fri, 14 Aug 2020 17:33:24 +0000 Subject: [PATCH 15/15] Formatting with black --- src/flownet/ert/_create_ert_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 7edb89063..4a1a460cd 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -120,7 +120,7 @@ def create_ert_setup( # pylint: disable=too-many-arguments if reference_simulation: path_ref_sim = pathlib.Path(reference_simulation).resolve() else: - path_ref_sim = pathlib.Path('.').resolve() + path_ref_sim = pathlib.Path(".").resolve() if prediction_setup: ert_config_file = output_folder / "pred_config.ert"