From bc263e8d2815e97fc0ca0b4c9fd8c5e6585134ec Mon Sep 17 00:00:00 2001 From: olelod Date: Sun, 23 Aug 2020 19:30:55 +0000 Subject: [PATCH 01/12] observation errors in config file, also passing the full config into create_ert_setup --- examples/norne_parameters.yml | 19 +- src/flownet/ahm/_assisted_history_matching.py | 33 ++- src/flownet/ahm/_run_ahm.py | 10 +- src/flownet/config_parser/_config_parser.py | 188 +++++++++++++----- src/flownet/ert/_create_ert_setup.py | 44 ++-- src/flownet/templates/ahm_config.ert.jinja2 | 16 +- .../templates/common_config.ert.jinja2 | 28 +-- .../templates/observations.ertobs.jinja2 | 8 +- .../templates/observations.yamlobs.jinja2 | 10 +- 9 files changed, 231 insertions(+), 125 deletions(-) diff --git a/examples/norne_parameters.yml b/examples/norne_parameters.yml index 3a6913966..e6efb7879 100644 --- a/examples/norne_parameters.yml +++ b/examples/norne_parameters.yml @@ -2,7 +2,24 @@ name: Norne flownet: data_source: - input_case: ../tests/data/norne/NORNE_ATW2013 + simulation: + input_case: ../tests/data/norne/NORNE_ATW2013 + vectors: + WTHP: + rel_error: 0.05 + min_error: 5 + WBHP: + rel_error: 0.05 + min_error: 10 + WOPR: + rel_error: 0.1 + min_error: 50 + WGPR: + rel_error: 0.1 + min_error: 100000 + WWPR: + rel_error: 0.1 + min_error: 50 phases: - oil - gas diff --git a/src/flownet/ahm/_assisted_history_matching.py b/src/flownet/ahm/_assisted_history_matching.py index 83ac5819d..b34c3b1d6 100644 --- a/src/flownet/ahm/_assisted_history_matching.py +++ b/src/flownet/ahm/_assisted_history_matching.py @@ -7,7 +7,7 @@ import re import shutil import subprocess -from typing import List, Dict, Optional, Tuple +from typing import List, Dict, Tuple import jinja2 import numpy as np @@ -38,9 +38,7 @@ def __init__( network: NetworkModel, schedule: Schedule, parameters: List[Parameter], - case_name: str, - ert_config: Dict, - random_seed: Optional[int] = None, + config: Dict, ): """ Initialize an Assisted History Matching Class @@ -49,18 +47,14 @@ def __init__( network: NetworkModel instance schedule: Schedule instance parameters: List of Parameter objects - case_name: Name of simulation case - ert_config: Dictionary containing information about queue (system, name, server and max_running) + config: Dictionary containing information about queue (system, name, server and max_running) and realizations (num_realizations, required_success_percent and max_runtime) - random_seed: Random seed to control reproducibility of FlowNet """ self._network: NetworkModel = network self._schedule: Schedule = schedule self._parameters: List[Parameter] = parameters - self._ert_config: dict = ert_config - self._case_name: str = case_name - self._random_seed: Optional[int] = random_seed + self._config: dict = config def create_ert_setup(self, args: argparse.Namespace, training_set_fraction: float): # pylint: disable=attribute-defined-outside-init @@ -82,9 +76,8 @@ def create_ert_setup(self, args: argparse.Namespace, training_set_fraction: floa args, self._network, self._schedule, - ert_config=self._ert_config, + config=self._config, parameters=self._parameters, - random_seed=self._random_seed, training_set_fraction=training_set_fraction, ) @@ -115,7 +108,7 @@ def run_ert(self, weights: List[float]): { "output_folder": self.output_folder, "iterations": range(len(weights) + 1), - "runpath": self._ert_config["runpath"], + "runpath": self._config["ert"]["runpath"], } ) ) @@ -134,7 +127,7 @@ def run_ert(self, weights: List[float]): error_files = glob.glob( str( self.output_folder - / self._ert_config["runpath"].replace("%d", "*") + / self._config["ert"]["runpath"].replace("%d", "*") / "ERROR" ) ) @@ -157,7 +150,7 @@ def report(self): f"Number of observations: {self._schedule.get_nr_observations(self._training_set_fraction):>20}" ) print( - f"Number of realizations: {self._ert_config['realizations'].num_realizations:>20}" + f"Number of realizations: {self._config['ert']['realizations'].num_realizations:>20}" ) distributions = { @@ -173,14 +166,14 @@ def report(self): for distribution in distributions: if distribution[0] == LogUniformDistribution: print( - "Loguniform".ljust((15)), + "Loguniform".ljust(15), f"{distribution[1]:16.8f}", f"{(distribution[2] - distribution[1]) / np.log(distribution[2] / distribution[1]):16.8f}", f"{distribution[2]:16.8f}", ) else: print( - "Uniform".ljust((15)), + "Uniform".ljust(15), f"{distribution[1]:16.8f}", f"{(distribution[2] + distribution[1]) / 2.0:16.8f}", f"{distribution[2]:16.8f}", @@ -197,7 +190,7 @@ def delete_simulation_output(): Nothing """ - parser = argparse.ArgumentParser(prog=("Delete simulation output.")) + parser = argparse.ArgumentParser(prog="Delete simulation output.") parser.add_argument( "ecl_base", type=str, help="Base name of the simulation DATA file" @@ -225,7 +218,7 @@ def _load_parameters(runpath: str) -> Tuple[int, Dict]: realization = int(re.findall(r"[0-9]+", runpath)[-2]) parameters = json.loads((pathlib.Path(runpath) / "parameters.json").read_text()) - return (realization, parameters["FLOWNET_PARAMETERS"]) + return realization, parameters["FLOWNET_PARAMETERS"] def save_iteration_parameters(): @@ -250,7 +243,7 @@ def save_iteration_parameters(): Nothing """ - parser = argparse.ArgumentParser(prog=("Save iteration parameters to a file.")) + parser = argparse.ArgumentParser(prog="Save iteration parameters to a file.") parser.add_argument("runpath", type=str, help="Path to the ERT runpath.") args = parser.parse_args() args.runpath = args.runpath.replace("%d", "*") diff --git a/src/flownet/ahm/_run_ahm.py b/src/flownet/ahm/_run_ahm.py index 54859ccad..335b858a3 100644 --- a/src/flownet/ahm/_run_ahm.py +++ b/src/flownet/ahm/_run_ahm.py @@ -143,7 +143,7 @@ def run_flownet_history_matching( # Load production and well coordinate data field_data = FlowData( - config.flownet.data_source.input_case, + config.flownet.data_source.simulation.input_case, perforation_handling_strategy=config.flownet.perforation_handling_strategy, ) df_production_data: pd.DataFrame = field_data.production @@ -153,7 +153,6 @@ def run_flownet_history_matching( df_fault_planes: Optional[ pd.DataFrame ] = field_data.faults if config.model_parameters.fault_mult else None - df_connections: pd.DataFrame = create_connections(df_coordinates, config) network = NetworkModel( @@ -392,12 +391,7 @@ def run_flownet_history_matching( parameters.append(FaultTransmissibility(fault_mult_dist_values, network)) ahm = AssistedHistoryMatching( - network, - schedule, - parameters, - case_name=config.name, - ert_config=config.ert._asdict(), - random_seed=config.flownet.random_seed, + network, schedule, parameters, config=config._asdict(), ) ahm.create_ert_setup( diff --git a/src/flownet/config_parser/_config_parser.py b/src/flownet/config_parser/_config_parser.py index d4862ad52..dd4509b02 100644 --- a/src/flownet/config_parser/_config_parser.py +++ b/src/flownet/config_parser/_config_parser.py @@ -29,9 +29,94 @@ def create_schema(_to_abs_path) -> Dict: "data_source": { MK.Type: types.NamedDict, MK.Content: { - "input_case": { - MK.Type: types.String, - MK.Transformation: _to_abs_path, + "simulation": { + MK.Type: types.NamedDict, + MK.Content: { + "input_case": { + MK.Type: types.String, + MK.Transformation: _to_abs_path, + }, + "vectors": { + MK.Type: types.NamedDict, + MK.Content: { + "WTHP": { + MK.Type: types.NamedDict, + MK.Content: { + "rel_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + "min_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + }, + }, + "WBHP": { + MK.Type: types.NamedDict, + MK.Content: { + "rel_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + "min_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + }, + }, + "WOPR": { + MK.Type: types.NamedDict, + MK.Content: { + "rel_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + "min_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + }, + }, + "WGPR": { + MK.Type: types.NamedDict, + MK.Content: { + "rel_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + "min_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + }, + }, + "WWPR": { + MK.Type: types.NamedDict, + MK.Content: { + "rel_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + "min_error": { + MK.Type: types.Number, + MK.Required: False, + MK.AllowNone: True, + }, + }, + }, + }, + }, + }, }, }, }, @@ -51,12 +136,12 @@ def create_schema(_to_abs_path) -> Dict: MK.Transformation: lambda names: [x.lower() for x in names], }, "cell_length": {MK.Type: types.Number}, - "training_set_end_date": {MK.Type: types.Date, MK.AllowNone: True}, + "training_set_end_date": {MK.Type: types.Date, MK.AllowNone: True,}, "training_set_fraction": { MK.Type: types.Number, MK.AllowNone: True, }, - "additional_flow_nodes": {MK.Type: types.Integer, MK.Default: 100}, + "additional_flow_nodes": {MK.Type: types.Integer, MK.Default: 100,}, "additional_node_candidates": { MK.Type: types.Integer, MK.Default: 1000, @@ -78,7 +163,7 @@ def create_schema(_to_abs_path) -> Dict: MK.Type: types.String, MK.Default: "output/runpath/realization-%d/iter-%d", }, - "enspath": {MK.Type: types.String, MK.Default: "output/storage"}, + "enspath": {MK.Type: types.String, MK.Default: "output/storage",}, "eclbase": { MK.Type: types.String, MK.Default: "./eclipse/model/FLOWNET_REALIZATION", @@ -101,7 +186,7 @@ def create_schema(_to_abs_path) -> Dict: MK.Type: types.Number, MK.Default: 20, }, - "max_runtime": {MK.Type: types.Integer, MK.Default: 300}, + "max_runtime": {MK.Type: types.Integer, MK.Default: 300,}, }, }, "simulator": { @@ -140,8 +225,8 @@ def create_schema(_to_abs_path) -> Dict: MK.Type: types.String, MK.Default: "[WOPR:BR-P-]", }, - "start": {MK.Type: types.String, MK.Default: "2001-04-01"}, - "end": {MK.Type: types.String, MK.Default: "2006-01-01"}, + "start": {MK.Type: types.String, MK.Default: "2001-04-01",}, + "end": {MK.Type: types.String, MK.Default: "2006-01-01",}, "outfile": { MK.Type: types.String, MK.Default: "analysis_metrics_iteration", @@ -199,8 +284,8 @@ def create_schema(_to_abs_path) -> Dict: "swirr": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -210,8 +295,8 @@ def create_schema(_to_abs_path) -> Dict: "swl": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -221,8 +306,8 @@ def create_schema(_to_abs_path) -> Dict: "swcr": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -243,8 +328,8 @@ def create_schema(_to_abs_path) -> Dict: "krwend": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -254,8 +339,8 @@ def create_schema(_to_abs_path) -> Dict: "krowend": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -265,8 +350,8 @@ def create_schema(_to_abs_path) -> Dict: "nw": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -276,8 +361,8 @@ def create_schema(_to_abs_path) -> Dict: "now": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -287,8 +372,8 @@ def create_schema(_to_abs_path) -> Dict: "sorg": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -298,8 +383,8 @@ def create_schema(_to_abs_path) -> Dict: "sgcr": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -309,15 +394,15 @@ def create_schema(_to_abs_path) -> Dict: "ng": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, }, }, "nog": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -327,8 +412,8 @@ def create_schema(_to_abs_path) -> Dict: "krgend": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -338,8 +423,8 @@ def create_schema(_to_abs_path) -> Dict: "krogend": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -356,7 +441,7 @@ def create_schema(_to_abs_path) -> Dict: MK.Default: "global", MK.Transformation: lambda name: name.lower(), }, - "datum_depth": {MK.Type: types.Number, MK.AllowNone: True}, + "datum_depth": {MK.Type: types.Number, MK.AllowNone: True,}, "datum_pressure": { MK.Type: types.NamedDict, MK.Content: { @@ -367,22 +452,22 @@ def create_schema(_to_abs_path) -> Dict: "owc_depth": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, }, }, "gwc_depth": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, }, }, "goc_depth": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, }, }, }, @@ -403,12 +488,12 @@ def create_schema(_to_abs_path) -> Dict: MK.Content: { "scheme": {MK.Type: types.String, MK.AllowNone: True}, "fraction": {MK.Type: types.Number, MK.AllowNone: True}, - "delta_depth": {MK.Type: types.Number, MK.AllowNone: True}, + "delta_depth": {MK.Type: types.Number, MK.AllowNone: True,}, "size_in_bulkvolumes": { MK.Type: types.NamedDict, MK.Content: { - "min": {MK.Type: types.Number, MK.AllowNone: True}, - "max": {MK.Type: types.Number, MK.AllowNone: True}, + "min": {MK.Type: types.Number, MK.AllowNone: True,}, + "max": {MK.Type: types.Number, MK.AllowNone: True,}, "loguniform": { MK.Type: types.Bool, MK.AllowNone: True, @@ -479,14 +564,15 @@ def _to_abs_path(path: Optional[str]) -> str: f"The {phase} phase is not a valid phase\n" f"The valid phases are 'oil', 'gas', 'water', 'disgas' and 'vapoil'" ) - if not set(["vapoil", "disgas"]).isdisjoint(config.flownet.phases) and not set( - ["oil", "gas"] - ).issubset(config.flownet.phases): + if not {"vapoil", "disgas"}.isdisjoint(config.flownet.phases) and not { + "oil", + "gas", + }.issubset(config.flownet.phases): raise ValueError( "The phases 'vapoil' and 'disgas' can not be defined without the phases 'oil' and 'gas'" ) - if set(["oil", "water"]).issubset(config.flownet.phases): + if {"oil", "water"}.issubset(config.flownet.phases): req_relp_parameters = req_relp_parameters + [ "scheme", "swirr", @@ -507,7 +593,7 @@ def _to_abs_path(path: Optional[str]) -> str: raise ValueError( "Ambiguous configuration input: OWC not properly specified. Min or max missing, or max < min." ) - if set(["oil", "gas"]).issubset(config.flownet.phases): + if {"oil", "gas"}.issubset(config.flownet.phases): req_relp_parameters = req_relp_parameters + [ "scheme", "swirr", @@ -612,7 +698,7 @@ def _to_abs_path(path: Optional[str]) -> str: for suffix in [".DATA", ".EGRID", ".UNRST", ".UNSMRY", ".SMSPEC"]: if ( - not pathlib.Path(config.flownet.data_source.input_case) + not pathlib.Path(config.flownet.data_source.simulation.input_case) .with_suffix(suffix) .is_file() ): diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index bb0688c1e..f440dc631 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -24,6 +24,7 @@ def _create_observation_file( schedule: Schedule, obs_file: pathlib.Path, + config: dict, training_set_fraction: float = 1, yaml: bool = False, ): @@ -36,6 +37,7 @@ def _create_observation_file( Args: schedule: FlowNet Schedule instance to create observations from. obs_file: Path to store the observation file. + config: Dictionary of config information training_set_fraction: Fraction of observations in schedule to use in training set yaml: Flag to indicate whether a yaml observation file is to be stored. False means ertobs. @@ -48,13 +50,28 @@ def _create_observation_file( if yaml: template = _TEMPLATE_ENVIRONMENT.get_template("observations.yamlobs.jinja2") with open(obs_file, "w") as fh: - fh.write(template.render({"schedule": schedule})) + fh.write( + template.render( + { + "schedule": schedule, + "error_config": config[ + "flownet" + ].data_source.simulation.vectors, + } + ) + ) else: template = _TEMPLATE_ENVIRONMENT.get_template("observations.ertobs.jinja2") with open(obs_file, "w") as fh: fh.write( template.render( - {"schedule": schedule, "num_training_dates": num_training_dates} + { + "schedule": schedule, + "error_config": config[ + "flownet" + ].data_source.simulation.vectors, + "num_training_dates": num_training_dates, + } ) ) @@ -91,9 +108,8 @@ def create_ert_setup( # pylint: disable=too-many-arguments args: argparse.Namespace, network, schedule: Schedule, - ert_config: dict, + config: dict, parameters=None, - random_seed=None, training_set_fraction: float = 1, prediction_setup: bool = False, ): @@ -140,12 +156,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments "pickled_schedule": output_folder.resolve() / "schedule.pickled", "pickled_parameters": output_folder.resolve() / "parameters.pickled", - "random_seed": random_seed, - "ert_config": ert_config, + "config": config, "debug": args.debug if hasattr(args, "debug") else False, - "pred_schedule_file": getattr( - ert_config, "pred_schedule_file", None - ), + "pred_schedule_file": getattr(config, "pred_schedule_file", None), } ) ) @@ -171,9 +184,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments ) static_path = ( - getattr(ert_config, "static_include_files") - if hasattr(ert_config, "static_include_files") - else ert_config["static_include_files"] + getattr(config["ert"], "static_include_files") + if hasattr(config["ert"], "static_include_files") + else config["ert"]["static_include_files"] ) shutil.copyfile( @@ -197,11 +210,14 @@ def create_ert_setup( # pylint: disable=too-many-arguments if parameters is not None: _create_observation_file( - schedule, output_folder / "observations.ertobs", training_set_fraction, + schedule, + output_folder / "observations.ertobs", + config, + training_set_fraction, ) _create_observation_file( - schedule, output_folder / "observations.yamlobs", yaml=True + schedule, output_folder / "observations.yamlobs", config, yaml=True ) _create_ert_parameter_file(parameters, output_folder) diff --git a/src/flownet/templates/ahm_config.ert.jinja2 b/src/flownet/templates/ahm_config.ert.jinja2 index 06b393a70..bcd4b35dd 100644 --- a/src/flownet/templates/ahm_config.ert.jinja2 +++ b/src/flownet/templates/ahm_config.ert.jinja2 @@ -3,15 +3,15 @@ INSTALL_JOB DELETE_IN_CURRENT_ITERATION ./DELETE_IN_CURRENT_ITERATION REFCASE /SYNTHETIC_REFCASE -DEFINE {{ ert_config.runpath }} -DEFINE {{ ert_config.eclbase }} +DEFINE {{ config.ert.runpath }} +DEFINE {{ config.ert.eclbase }} DEFINE ./parameters.json -DEFINE {{ ert_config.yamlobs }} -DEFINE {{ ert_config.analysis.metric }} -DEFINE {{ ert_config.analysis.quantity }} -DEFINE {{ ert_config.analysis.start }} -DEFINE {{ ert_config.analysis.end }} -DEFINE {{ ert_config.analysis.outfile }} +DEFINE {{ config.ert.yamlobs }} +DEFINE {{ config.ert.analysis.metric }} +DEFINE {{ config.ert.analysis.quantity }} +DEFINE {{ config.ert.analysis.start }} +DEFINE {{ config.ert.analysis.end }} +DEFINE {{ config.ert.analysis.outfile }} INSTALL_JOB CREATE_FLOWNET_MODEL ./CREATE_FLOWNET_MODEL {%- if not debug: %} diff --git a/src/flownet/templates/common_config.ert.jinja2 b/src/flownet/templates/common_config.ert.jinja2 index cee224603..c14bcba3b 100644 --- a/src/flownet/templates/common_config.ert.jinja2 +++ b/src/flownet/templates/common_config.ert.jinja2 @@ -1,23 +1,23 @@ -{% if random_seed is not none: -%} -RANDOM_SEED {{ random_seed }} +{% if config.flownet.random_seed is not none: -%} +RANDOM_SEED {{config.flownet.random_seed }} {%- endif %} -NUM_REALIZATIONS {{ ert_config.realizations.num_realizations }} -MIN_REALIZATIONS {{ ert_config.realizations.required_success_percent }}% +NUM_REALIZATIONS {{ config.ert.realizations.num_realizations }} +MIN_REALIZATIONS {{ config.ert.realizations.required_success_percent }}% STOP_LONG_RUNNING FALSE -MAX_RUNTIME {{ ert_config.realizations.max_runtime }} +MAX_RUNTIME {{ config.ert.realizations.max_runtime }} -QUEUE_SYSTEM {{ ert_config.queue.system }} -{%- if ert_config.queue.system == "LSF" %} -{% if ert_config.simulator.version is not none: -%} LSF_SERVER {{ ert_config.queue.server }} {%- endif %} -QUEUE_OPTION {{ ert_config.queue.system }} LSF_QUEUE {{ ert_config.queue.name }} +QUEUE_SYSTEM {{ config.ert.queue.system }} +{%- if config.ert.queue.system == "LSF" %} +{% if config.ert.simulator.version is not none: -%} LSF_SERVER {{ config.ert.queue.server }} {%- endif %} +QUEUE_OPTION {{ config.ert.queue.system }} LSF_QUEUE {{ config.ert.queue.name }} {%- endif %} -QUEUE_OPTION {{ ert_config.queue.system }} MAX_RUNNING {{ ert_config.queue.max_running }} +QUEUE_OPTION {{ config.ert.queue.system }} MAX_RUNNING {{ config.ert.queue.max_running }} -RUNPATH {{ ert_config.runpath }} -ENSPATH {{ ert_config.enspath }} +RUNPATH {{ config.ert.runpath }} +ENSPATH {{ config.ert.enspath }} -ECLBASE {{ ert_config.eclbase }} +ECLBASE {{ config.ert.eclbase }} DEFINE {{ pickled_network }} DEFINE {{ pickled_schedule }} @@ -33,4 +33,4 @@ FORWARD_MODEL COPY_FILE(=/PROPS.inc, =./eclipse/model/inc FORWARD_MODEL COPY_FILE(=/SOLUTION.inc, =./eclipse/model/include/SOLUTION.inc) FORWARD_MODEL COPY_FILE(=/SCHEDULE.inc, =./eclipse/model/include/SCHEDULE.inc) -SIMULATION_JOB {{ ert_config.simulator.name.upper() }} {%- if ert_config.simulator.version is not none: -%}"--version={{ ert_config.simulator.version }}"{%- endif %} +SIMULATION_JOB {{ config.ert.simulator.name.upper() }} {%- if config.ert.simulator.version is not none: -%}"--version={{ config.ert.simulator.version }}"{%- endif %} diff --git a/src/flownet/templates/observations.ertobs.jinja2 b/src/flownet/templates/observations.ertobs.jinja2 index b105ea34e..c35f561ee 100644 --- a/src/flownet/templates/observations.ertobs.jinja2 +++ b/src/flownet/templates/observations.ertobs.jinja2 @@ -9,16 +9,16 @@ {%- set date_formatted = date.strftime('%d/%m/%Y') %} {%- if not isnan(kw.oil_rate): -%} -SUMMARY_OBSERVATION WOPR_{{ kw.well_name }}_{{ index_name }} { VALUE = {{ kw.oil_rate }}; ERROR = {{ [0.1 * kw.oil_rate, 50] | max }}; DATE = {{ date_formatted }}; KEY = WOPR:{{ kw.well_name }}; }; +SUMMARY_OBSERVATION WOPR_{{ kw.well_name }}_{{ index_name }} { VALUE = {{ kw.oil_rate }}; ERROR = {{ [error_config.WOPR.rel_error * kw.oil_rate, error_config.WOPR.min_error] | max }}; DATE = {{ date_formatted }}; KEY = WOPR:{{ kw.well_name }}; }; {%- endif %} {%- if not isnan(kw.gas_rate): %} -SUMMARY_OBSERVATION WGPR_{{ kw.well_name }}_{{ index_name }} { VALUE = {{ kw.gas_rate }}; ERROR = {{ [0.1 * kw.gas_rate, 100000] | max }}; DATE = {{ date_formatted }}; KEY = WGPR:{{ kw.well_name }}; }; +SUMMARY_OBSERVATION WGPR_{{ kw.well_name }}_{{ index_name }} { VALUE = {{ kw.gas_rate }}; ERROR = {{ [error_config.WGPR.rel_error * kw.gas_rate, error_config.WGPR.min_error] | max }}; DATE = {{ date_formatted }}; KEY = WGPR:{{ kw.well_name }}; }; {%- endif %} {%- if not isnan(kw.bhp): %} -SUMMARY_OBSERVATION WBHP_{{ kw.well_name }}_{{ index_name }} { VALUE = {{ kw.bhp }}; ERROR = {{ [0.05 * kw.bhp, 10] | max }}; DATE = {{ date_formatted }}; KEY = WBHP:{{ kw.well_name }}; }; +SUMMARY_OBSERVATION WBHP_{{ kw.well_name }}_{{ index_name }} { VALUE = {{ kw.bhp }}; ERROR = {{ [error_config.WBHP.rel_error * kw.bhp, error_config.WBHP.min_error] | max }}; DATE = {{ date_formatted }}; KEY = WBHP:{{ kw.well_name }}; }; {%- endif %} {%- if not isnan(kw.thp): %} -SUMMARY_OBSERVATION WTHP_{{ kw.well_name }}_{{ index_name }} { VALUE = {{ kw.thp }}; ERROR = {{ [0.05 * kw.thp, 5] | max }}; DATE = {{ date_formatted }}; KEY = WTHP:{{ kw.well_name }}; }; +SUMMARY_OBSERVATION WTHP_{{ kw.well_name }}_{{ index_name }} { VALUE = {{ kw.thp }}; ERROR = {{ [error_config.WTHP.rel_error * kw.thp, error_config.WTHP.min_error] | max }}; DATE = {{ date_formatted }}; KEY = WTHP:{{ kw.well_name }}; }; {%- endif %} {%- endif %} diff --git a/src/flownet/templates/observations.yamlobs.jinja2 b/src/flownet/templates/observations.yamlobs.jinja2 index 90a6678bd..4f04ff37e 100644 --- a/src/flownet/templates/observations.yamlobs.jinja2 +++ b/src/flownet/templates/observations.yamlobs.jinja2 @@ -7,7 +7,7 @@ smry: {%- elif not loop.first: %} - date: {{ kw.date.strftime('%Y-%m-%d') }} value: {{ kw.oil_rate }} - error: {{ [0.1 * kw.oil_rate, 50] | max }} + error: {{ [error_config.WOPR.rel_error * kw.oil_rate, error_config.WOPR.min_error] | max }} {%- endif -%} {%- endfor %} {%- for kw in schedule.get_keywords(kw_class="WCONHIST", well_name=well_name, ignore_nan="gas_rate") -%} @@ -17,7 +17,7 @@ smry: {%- elif not loop.first: %} - date: {{ kw.date.strftime('%Y-%m-%d') }} value: {{ kw.gas_rate }} - error: {{ [0.1 * kw.gas_rate, 100000] | max }} + error: {{ [error_config.WGPR.rel_error * kw.oil_rate, error_config.WGPR.min_error] | max }} {%- endif -%} {%- endfor %} {%- for kw in schedule.get_keywords(kw_class="WCONHIST", well_name=well_name, ignore_nan="water_rate") -%} @@ -27,7 +27,7 @@ smry: {%- elif not loop.first: %} - date: {{ kw.date.strftime('%Y-%m-%d') }} value: {{ kw.water_rate }} - error: {{ [0.1 * kw.water_rate, 50] | max }} + error: {{ [error_config.WWPR.rel_error * kw.oil_rate, error_config.WWPR.min_error] | max }} {%- endif -%} {%- endfor %} {%- for kw in schedule.get_keywords(kw_class="WCONHIST", well_name=well_name, ignore_nan="bhp") -%} @@ -37,7 +37,7 @@ smry: {%- elif not loop.first: %} - date: {{ kw.date.strftime('%Y-%m-%d') }} value: {{ kw.bhp }} - error: {{ [0.05 * kw.bhp, 10] | max }} + error: {{ [error_config.WBHP.rel_error * kw.oil_rate, error_config.WBHP.min_error] | max }} {%- endif -%} {%- endfor %} {%- for kw in schedule.get_keywords(kw_class="WCONHIST", well_name=well_name, ignore_nan="thp") -%} @@ -47,7 +47,7 @@ smry: {%- elif not loop.first: %} - date: {{ kw.date.strftime('%Y-%m-%d') }} value: {{ kw.thp }} - error: {{ [0.05 * kw.thp, 5] | max }} + error: {{ [error_config.WTHP.rel_error * kw.oil_rate, error_config.WTHP.min_error] | max }} {%- endif -%} {%- endfor %} {%- endfor -%} From 42ad6a47aa053250a432225f30c3d15ab746277c Mon Sep 17 00:00:00 2001 From: olelod Date: Mon, 24 Aug 2020 07:40:16 +0000 Subject: [PATCH 02/12] some fixes --- src/flownet/ahm/_assisted_history_matching.py | 2 +- src/flownet/config_parser/_config_parser.py | 6 +++--- src/flownet/ert/_create_ert_setup.py | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/flownet/ahm/_assisted_history_matching.py b/src/flownet/ahm/_assisted_history_matching.py index 3e021ed27..c388226a7 100644 --- a/src/flownet/ahm/_assisted_history_matching.py +++ b/src/flownet/ahm/_assisted_history_matching.py @@ -135,7 +135,7 @@ def report(self): f"Number of observations: {self._schedule.get_nr_observations(self._training_set_fraction):>20}" ) print( - f"Number of realizations: {self._config["ert"].realizations.num_realizations:>20}" + f"Number of realizations: {self._config['ert'].realizations.num_realizations:>20}" ) distributions = { diff --git a/src/flownet/config_parser/_config_parser.py b/src/flownet/config_parser/_config_parser.py index 4720fb047..0299e39e9 100644 --- a/src/flownet/config_parser/_config_parser.py +++ b/src/flownet/config_parser/_config_parser.py @@ -691,9 +691,9 @@ def _to_abs_path(path: Optional[str]) -> str: ) for suffix in [".DATA", ".EGRID", ".UNRST", ".UNSMRY", ".SMSPEC"]: - input_file = pathlib.Path(config.flownet.data_source.simulation.input_case).with_suffix( - suffix - ) + input_file = pathlib.Path( + config.flownet.data_source.simulation.input_case + ).with_suffix(suffix) if not input_file.is_file(): raise ValueError(f"The file {input_file} does not exist") diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index b63c5ca72..c43419650 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -132,7 +132,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments # Derive absolute path to reference simulation case if config["flownet"].data_source.simulation.input_case: - path_ref_sim = pathlib.Path(config["flownet"].data_source.simulation.input_case).resolve() + path_ref_sim = pathlib.Path( + config["flownet"].data_source.simulation.input_case + ).resolve() else: path_ref_sim = pathlib.Path(".").resolve() From 9bc9447a51d764014b691450ac0bd510e676d868 Mon Sep 17 00:00:00 2001 From: olelod Date: Mon, 24 Aug 2020 07:43:56 +0000 Subject: [PATCH 03/12] some fixes --- src/flownet/ahm/_assisted_history_matching.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/flownet/ahm/_assisted_history_matching.py b/src/flownet/ahm/_assisted_history_matching.py index c388226a7..401659891 100644 --- a/src/flownet/ahm/_assisted_history_matching.py +++ b/src/flownet/ahm/_assisted_history_matching.py @@ -6,7 +6,6 @@ import pathlib import re import shutil -import subprocess from typing import List, Dict, Tuple import jinja2 From cacb5c61fe7e8fc10066cf276cdd87908f478c01 Mon Sep 17 00:00:00 2001 From: olelod Date: Mon, 24 Aug 2020 12:49:46 +0000 Subject: [PATCH 04/12] changing branch on flownet-testdata --- .github/workflows/flownet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flownet.yml b/.github/workflows/flownet.yml index 31904f47b..97fe0f4ba 100644 --- a/.github/workflows/flownet.yml +++ b/.github/workflows/flownet.yml @@ -56,7 +56,7 @@ jobs: TESTDATA_REPO_OWNER: equinor # If you want the CI to (temporarily) run against another branch than master, # change the value her from "master" to the relevant branch name. - TESTDATA_REPO_BRANCH: master + TESTDATA_REPO_BRANCH: observation-errors-in-config-yaml run: | git clone --depth 1 --branch $TESTDATA_REPO_BRANCH https://github.com/$TESTDATA_REPO_OWNER/flownet-testdata.git if [ -f "$INPUT_MODEL_FOLDER/${{ matrix.flownet-model }}.tar.gz" ]; then From 2ad19098e7bd6b8de6417c43467cdd08ed9786f0 Mon Sep 17 00:00:00 2001 From: olelod Date: Tue, 25 Aug 2020 11:09:22 +0000 Subject: [PATCH 05/12] fixup --- src/flownet/ert/_create_ert_setup.py | 41 +++++++++++-------- src/flownet/prediction/_run_pred.py | 2 +- .../templates/common_config.ert.jinja2 | 4 +- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index c43419650..b880237a0 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -130,13 +130,15 @@ def create_ert_setup( # pylint: disable=too-many-arguments output_folder = pathlib.Path(args.output_folder) os.makedirs(output_folder, exist_ok=True) - # Derive absolute path to reference simulation case - if config["flownet"].data_source.simulation.input_case: - path_ref_sim = pathlib.Path( - config["flownet"].data_source.simulation.input_case - ).resolve() - else: - path_ref_sim = pathlib.Path(".").resolve() + + if not prediction_setup: + # Derive absolute path to reference simulation case + if config["flownet"].data_source.simulation.input_case: + path_ref_sim = pathlib.Path( + config["flownet"].data_source.simulation.input_case + ).resolve() + else: + path_ref_sim = pathlib.Path(".").resolve() if prediction_setup: ert_config_file = output_folder / "pred_config.ert" @@ -165,7 +167,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments "pickled_parameters": output_folder.resolve() / "parameters.pickled", "config": config, - "reference_simulation": path_ref_sim, + "random_seed": config["flownet"].random_seed if not prediction_setup else None, + "perforation_strategy": config["flownet"].perforation_strategy if not prediction_setup else None, + "reference_simulation": path_ref_sim if not prediction_setup else None, "debug": args.debug if hasattr(args, "debug") else False, "pred_schedule_file": getattr(config, "pred_schedule_file", None), } @@ -217,17 +221,18 @@ def create_ert_setup( # pylint: disable=too-many-arguments # Otherwise create an empty one. (output_folder / f"{section}.inc").touch() - if parameters is not None: - _create_observation_file( - schedule, - output_folder / "observations.ertobs", - config, - training_set_fraction, - ) + if not prediction_setup: + if parameters is not None: + _create_observation_file( + schedule, + output_folder / "observations.ertobs", + config, + training_set_fraction, + ) - _create_observation_file( - schedule, output_folder / "observations.yamlobs", config, yaml=True - ) + _create_observation_file( + schedule, output_folder / "observations.yamlobs", config, yaml=True + ) _create_ert_parameter_file(parameters, output_folder) diff --git a/src/flownet/prediction/_run_pred.py b/src/flownet/prediction/_run_pred.py index 413f9392e..df6d1f832 100644 --- a/src/flownet/prediction/_run_pred.py +++ b/src/flownet/prediction/_run_pred.py @@ -33,7 +33,7 @@ def run_flownet_prediction(config, args): parameters = pickle.load(fh) create_ert_setup( - args, network, schedule, config.ert, parameters, prediction_setup=True, + args, network, schedule, config._asdict(), parameters, prediction_setup=True, ) shutil.copyfile( args.ahm_folder / "parameters_iteration-latest.parquet.gzip", diff --git a/src/flownet/templates/common_config.ert.jinja2 b/src/flownet/templates/common_config.ert.jinja2 index c14bcba3b..4064b11dd 100644 --- a/src/flownet/templates/common_config.ert.jinja2 +++ b/src/flownet/templates/common_config.ert.jinja2 @@ -1,5 +1,5 @@ -{% if config.flownet.random_seed is not none: -%} -RANDOM_SEED {{config.flownet.random_seed }} +{% if random_seed is not none: -%} +RANDOM_SEED {{ random_seed }} {%- endif %} NUM_REALIZATIONS {{ config.ert.realizations.num_realizations }} From f7c2ea85914b612ff25b66b2c84f211a09b58c3e Mon Sep 17 00:00:00 2001 From: olelod Date: Tue, 25 Aug 2020 11:10:49 +0000 Subject: [PATCH 06/12] black --- src/flownet/ert/_create_ert_setup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index b880237a0..cea5b8f4e 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -130,7 +130,6 @@ def create_ert_setup( # pylint: disable=too-many-arguments output_folder = pathlib.Path(args.output_folder) os.makedirs(output_folder, exist_ok=True) - if not prediction_setup: # Derive absolute path to reference simulation case if config["flownet"].data_source.simulation.input_case: @@ -167,9 +166,15 @@ def create_ert_setup( # pylint: disable=too-many-arguments "pickled_parameters": output_folder.resolve() / "parameters.pickled", "config": config, - "random_seed": config["flownet"].random_seed if not prediction_setup else None, - "perforation_strategy": config["flownet"].perforation_strategy if not prediction_setup else None, - "reference_simulation": path_ref_sim if not prediction_setup else None, + "random_seed": config["flownet"].random_seed + if not prediction_setup + else None, + "perforation_strategy": config["flownet"].perforation_strategy + if not prediction_setup + else None, + "reference_simulation": path_ref_sim + if not prediction_setup + else None, "debug": args.debug if hasattr(args, "debug") else False, "pred_schedule_file": getattr(config, "pred_schedule_file", None), } From 5fcd4e698bb1ccd435d0a8bd2611d7013cb81f96 Mon Sep 17 00:00:00 2001 From: olelod Date: Tue, 25 Aug 2020 11:21:58 +0000 Subject: [PATCH 07/12] minor fix --- src/flownet/ert/_create_ert_setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index cea5b8f4e..b664c537b 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -169,7 +169,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments "random_seed": config["flownet"].random_seed if not prediction_setup else None, - "perforation_strategy": config["flownet"].perforation_strategy + "perforation_strategy": config[ + "flownet" + ].perforation_handling_strategy if not prediction_setup else None, "reference_simulation": path_ref_sim From a3f057d6cf5d5c15ebbde881168569427c721509 Mon Sep 17 00:00:00 2001 From: olelod Date: Tue, 25 Aug 2020 13:56:58 +0000 Subject: [PATCH 08/12] changes after review --- src/flownet/ahm/_assisted_history_matching.py | 12 +++--- src/flownet/ahm/_run_ahm.py | 2 +- src/flownet/ert/_create_ert_setup.py | 38 +++++++++---------- src/flownet/prediction/_run_pred.py | 2 +- 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/flownet/ahm/_assisted_history_matching.py b/src/flownet/ahm/_assisted_history_matching.py index 401659891..c12ad5b70 100644 --- a/src/flownet/ahm/_assisted_history_matching.py +++ b/src/flownet/ahm/_assisted_history_matching.py @@ -7,7 +7,7 @@ import re import shutil from typing import List, Dict, Tuple - +from configsuite import ConfigSuite import jinja2 import numpy as np import pandas as pd @@ -37,7 +37,7 @@ def __init__( network: NetworkModel, schedule: Schedule, parameters: List[Parameter], - config: Dict, + config: ConfigSuite.snapshot, ): """ Initialize an Assisted History Matching Class @@ -52,7 +52,7 @@ def __init__( self._schedule: Schedule = schedule self._parameters: List[Parameter] = parameters - self._config: dict = config + self._config: ConfigSuite.snapshot = config def create_ert_setup(self, args: argparse.Namespace, training_set_fraction: float): # pylint: disable=attribute-defined-outside-init @@ -106,7 +106,7 @@ def run_ert(self, weights: List[float]): { "output_folder": self.output_folder, "iterations": range(len(weights) + 1), - "runpath": self._config["ert"].runpath, + "runpath": self._config.ert.runpath, } ) ) @@ -114,7 +114,7 @@ def run_ert(self, weights: List[float]): run_ert_subprocess( f"ert es_mda --weights {','.join(map(str, weights))!r} ahm_config.ert", cwd=self.output_folder, - runpath=self._config["ert"].runpath, + runpath=self._config.ert.runpath, ) def report(self): @@ -134,7 +134,7 @@ def report(self): f"Number of observations: {self._schedule.get_nr_observations(self._training_set_fraction):>20}" ) print( - f"Number of realizations: {self._config['ert'].realizations.num_realizations:>20}" + f"Number of realizations: {self._config.ert.realizations.num_realizations:>20}" ) distributions = { diff --git a/src/flownet/ahm/_run_ahm.py b/src/flownet/ahm/_run_ahm.py index 19c31e79b..1343c774d 100644 --- a/src/flownet/ahm/_run_ahm.py +++ b/src/flownet/ahm/_run_ahm.py @@ -394,7 +394,7 @@ def run_flownet_history_matching( parameters.append(FaultTransmissibility(fault_mult_dist_values, network)) ahm = AssistedHistoryMatching( - network, schedule, parameters, config=config._asdict(), + network, schedule, parameters, config, ) ahm.create_ert_setup( diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index b664c537b..85e015a09 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -4,7 +4,7 @@ import pickle import shutil from typing import List - +from configsuite import ConfigSuite import jinja2 import numpy as np @@ -24,7 +24,7 @@ def _create_observation_file( schedule: Schedule, obs_file: pathlib.Path, - config: dict, + config: ConfigSuite.snapshot, training_set_fraction: float = 1, yaml: bool = False, ): @@ -37,7 +37,7 @@ def _create_observation_file( Args: schedule: FlowNet Schedule instance to create observations from. obs_file: Path to store the observation file. - config: Dictionary of config information + config: Information from the FlowNet config yaml training_set_fraction: Fraction of observations in schedule to use in training set yaml: Flag to indicate whether a yaml observation file is to be stored. False means ertobs. @@ -54,9 +54,7 @@ def _create_observation_file( template.render( { "schedule": schedule, - "error_config": config[ - "flownet" - ].data_source.simulation.vectors, + "error_config": config.flownet.data_source.simulation.vectors, } ) ) @@ -67,9 +65,7 @@ def _create_observation_file( template.render( { "schedule": schedule, - "error_config": config[ - "flownet" - ].data_source.simulation.vectors, + "error_config": config.flownet.data_source.simulation.vectors, "num_training_dates": num_training_dates, } ) @@ -84,7 +80,7 @@ def _create_ert_parameter_file( and outputs them in an ert parameter definition file Args: - parameters: List with Paratemers + parameters: List with Parameters output_folder: Path to the output_folder Returns: @@ -108,7 +104,7 @@ def create_ert_setup( # pylint: disable=too-many-arguments args: argparse.Namespace, network, schedule: Schedule, - config: dict, + config: ConfigSuite.snapshot, parameters=None, training_set_fraction: float = 1, prediction_setup: bool = False, @@ -119,7 +115,11 @@ def create_ert_setup( # pylint: disable=too-many-arguments Args: schedule: FlowNet Schedule instance to create ERT setup from args: Arguments given to FlowNet at execution + network: FlowNet network instance + config: Information from the FlowNet config yaml + parameters: List with parameters (default = None) training_set_fraction: Fraction of observations to be used for model training (default = 1) + prediction_setup: Set to true if it is a prediction run (default = False) Returns: Nothing @@ -132,9 +132,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments if not prediction_setup: # Derive absolute path to reference simulation case - if config["flownet"].data_source.simulation.input_case: + if config.flownet.data_source.simulation.input_case: path_ref_sim = pathlib.Path( - config["flownet"].data_source.simulation.input_case + config.flownet.data_source.simulation.input_case ).resolve() else: path_ref_sim = pathlib.Path(".").resolve() @@ -166,12 +166,10 @@ def create_ert_setup( # pylint: disable=too-many-arguments "pickled_parameters": output_folder.resolve() / "parameters.pickled", "config": config, - "random_seed": config["flownet"].random_seed + "random_seed": config.flownet.random_seed if not prediction_setup else None, - "perforation_strategy": config[ - "flownet" - ].perforation_handling_strategy + "perforation_strategy": config.flownet.perforation_handling_strategy if not prediction_setup else None, "reference_simulation": path_ref_sim @@ -204,9 +202,9 @@ def create_ert_setup( # pylint: disable=too-many-arguments ) static_path = ( - getattr(config["ert"], "static_include_files") - if hasattr(config["ert"], "static_include_files") - else config["ert"]["static_include_files"] + getattr(config.ert, "static_include_files") + if hasattr(config.ert, "static_include_files") + else config.ert.static_include_files ) shutil.copyfile( diff --git a/src/flownet/prediction/_run_pred.py b/src/flownet/prediction/_run_pred.py index df6d1f832..4c177eef5 100644 --- a/src/flownet/prediction/_run_pred.py +++ b/src/flownet/prediction/_run_pred.py @@ -33,7 +33,7 @@ def run_flownet_prediction(config, args): parameters = pickle.load(fh) create_ert_setup( - args, network, schedule, config._asdict(), parameters, prediction_setup=True, + args, network, schedule, config, parameters, prediction_setup=True, ) shutil.copyfile( args.ahm_folder / "parameters_iteration-latest.parquet.gzip", From 05b8e9d5f4ef79b039052416a3cadc0e85f0239e Mon Sep 17 00:00:00 2001 From: olelod Date: Wed, 26 Aug 2020 10:49:42 +0000 Subject: [PATCH 09/12] changes after code review --- src/flownet/ahm/_assisted_history_matching.py | 3 +- src/flownet/ahm/_run_ahm.py | 4 +- src/flownet/ert/_create_ert_setup.py | 43 ++++++++++--------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/flownet/ahm/_assisted_history_matching.py b/src/flownet/ahm/_assisted_history_matching.py index c12ad5b70..a986608c1 100644 --- a/src/flownet/ahm/_assisted_history_matching.py +++ b/src/flownet/ahm/_assisted_history_matching.py @@ -7,6 +7,7 @@ import re import shutil from typing import List, Dict, Tuple + from configsuite import ConfigSuite import jinja2 import numpy as np @@ -46,7 +47,7 @@ def __init__( network: NetworkModel instance schedule: Schedule instance parameters: List of Parameter objects - config: Dictionary containing information about queue (system, name, server and max_running) + config: Information from the FlowNet config YAML """ self._network: NetworkModel = network self._schedule: Schedule = schedule diff --git a/src/flownet/ahm/_run_ahm.py b/src/flownet/ahm/_run_ahm.py index 1343c774d..913a0a947 100644 --- a/src/flownet/ahm/_run_ahm.py +++ b/src/flownet/ahm/_run_ahm.py @@ -393,9 +393,7 @@ def run_flownet_history_matching( if isinstance(network.faults, dict): parameters.append(FaultTransmissibility(fault_mult_dist_values, network)) - ahm = AssistedHistoryMatching( - network, schedule, parameters, config, - ) + ahm = AssistedHistoryMatching(network, schedule, parameters, config,) ahm.create_ert_setup( args=args, training_set_fraction=_find_training_set_fraction(schedule, config), diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 85e015a09..6b0b417be 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -4,6 +4,7 @@ import pickle import shutil from typing import List + from configsuite import ConfigSuite import jinja2 import numpy as np @@ -157,28 +158,30 @@ def create_ert_setup( # pylint: disable=too-many-arguments with open(output_folder / "parameters.pickled", "wb") as fh: pickle.dump(parameters, fh) + configuration = { + "pickled_network": output_folder.resolve() / "network.pickled", + "pickled_schedule": output_folder.resolve() / "schedule.pickled", + "pickled_parameters": output_folder.resolve() / "parameters.pickled", + "config": config, + "random_seed": None, + "reference_simulation": None, + "perforation_strategy": None, + "debug": args.debug if hasattr(args, "debug") else False, + "pred_schedule_file": getattr(config.ert, "pred_schedule_file", None), + } + + if not prediction_setup: + configuration.update( + { + "random_seed": config.flownet.random_seed, + "reference_simulation": path_ref_sim, + "perforation_strategy": config.flownet.perforation_handling_strategy, + } + ) + with open(ert_config_file, "w") as fh: # type: ignore[assignment] fh.write( # type: ignore[call-overload] - template.render( - { - "pickled_network": output_folder.resolve() / "network.pickled", - "pickled_schedule": output_folder.resolve() / "schedule.pickled", - "pickled_parameters": output_folder.resolve() - / "parameters.pickled", - "config": config, - "random_seed": config.flownet.random_seed - if not prediction_setup - else None, - "perforation_strategy": config.flownet.perforation_handling_strategy - if not prediction_setup - else None, - "reference_simulation": path_ref_sim - if not prediction_setup - else None, - "debug": args.debug if hasattr(args, "debug") else False, - "pred_schedule_file": getattr(config, "pred_schedule_file", None), - } - ) + template.render(configuration) ) shutil.copyfile( From abb5d2d9c7d81a6e865e9239a3eec8d54ad99846 Mon Sep 17 00:00:00 2001 From: olelod Date: Wed, 26 Aug 2020 11:03:08 +0000 Subject: [PATCH 10/12] removed a couple of unnecessarylines --- src/flownet/ert/_create_ert_setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/flownet/ert/_create_ert_setup.py b/src/flownet/ert/_create_ert_setup.py index 6b0b417be..d71e004fd 100644 --- a/src/flownet/ert/_create_ert_setup.py +++ b/src/flownet/ert/_create_ert_setup.py @@ -164,8 +164,6 @@ def create_ert_setup( # pylint: disable=too-many-arguments "pickled_parameters": output_folder.resolve() / "parameters.pickled", "config": config, "random_seed": None, - "reference_simulation": None, - "perforation_strategy": None, "debug": args.debug if hasattr(args, "debug") else False, "pred_schedule_file": getattr(config.ert, "pred_schedule_file", None), } From e97a737bc5b0bcaa9402e005b39ef69135e4a97c Mon Sep 17 00:00:00 2001 From: olelod Date: Wed, 26 Aug 2020 13:31:24 +0000 Subject: [PATCH 11/12] changing to master branch on flownet-testdata --- .github/workflows/flownet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flownet.yml b/.github/workflows/flownet.yml index d8f37159c..1896925c6 100644 --- a/.github/workflows/flownet.yml +++ b/.github/workflows/flownet.yml @@ -56,7 +56,7 @@ jobs: TESTDATA_REPO_OWNER: equinor # If you want the CI to (temporarily) run against another branch than master, # change the value her from "master" to the relevant branch name. - TESTDATA_REPO_BRANCH: observation-errors-in-config-yaml + TESTDATA_REPO_BRANCH: master run: | git clone --depth 1 --branch $TESTDATA_REPO_BRANCH https://github.com/$TESTDATA_REPO_OWNER/flownet-testdata.git if [ -f "$INPUT_MODEL_FOLDER/${{ matrix.flownet-model }}.tar.gz" ]; then From 9e46c5b28c9da615acbb499a858b4cb4f657f5ac Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" <9119793+wouterjdb@users.noreply.github.com> Date: Wed, 26 Aug 2020 15:55:18 +0200 Subject: [PATCH 12/12] Update src/flownet/ahm/_assisted_history_matching.py --- src/flownet/ahm/_assisted_history_matching.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/flownet/ahm/_assisted_history_matching.py b/src/flownet/ahm/_assisted_history_matching.py index a986608c1..884f7c6a6 100644 --- a/src/flownet/ahm/_assisted_history_matching.py +++ b/src/flownet/ahm/_assisted_history_matching.py @@ -52,7 +52,6 @@ def __init__( self._network: NetworkModel = network self._schedule: Schedule = schedule self._parameters: List[Parameter] = parameters - self._config: ConfigSuite.snapshot = config def create_ert_setup(self, args: argparse.Namespace, training_set_fraction: float):