diff --git a/src/jade/__main__.py b/src/jade/__main__.py index 9ede1688..2ab7e1b9 100644 --- a/src/jade/__main__.py +++ b/src/jade/__main__.py @@ -7,8 +7,12 @@ def main(): parser = argparse.ArgumentParser(description="JADE V&V") parser.add_argument("--run", help="run benchmarks", action="store_true") parser.add_argument("--raw", help="process raw results", action="store_true") - parser.add_argument("--pp", help="perform complete post-process of the results", action="store_true") - parser.add_argument("--gui", help="open the run configuration GUI", action="store_true") + parser.add_argument( + "--pp", help="perform complete post-process of the results", action="store_true" + ) + parser.add_argument( + "--gui", help="open the run configuration GUI", action="store_true" + ) args = parser.parse_args() @@ -22,7 +26,7 @@ def main(): if args.raw: app.raw_process() if args.pp: - raise NotImplementedError("JADE Post-processing not re-implemented yet") + app.post_process() if __name__ == "__main__": diff --git a/src/jade/app/app.py b/src/jade/app/app.py index f3dde16e..babb299e 100644 --- a/src/jade/app/app.py +++ b/src/jade/app/app.py @@ -11,15 +11,17 @@ from tqdm import tqdm import jade.resources as res +from jade import resources from jade.app.fetch import fetch_iaea_inputs from jade.config.paths_tree import PathsTree from jade.config.pp_config import PostProcessConfig from jade.config.raw_config import ConfigRawProcessor from jade.config.run_config import RunConfig from jade.config.status import GlobalStatus -from jade.gui.config_gui import ConfigGUI +from jade.gui.run_config_gui import ConfigGUI from jade.helper.aux_functions import PathLike, get_code_lib, print_code_lib -from jade.helper.constants import CODE, FIRST_INITIALIZATION, JADE_TITLE +from jade.helper.constants import CODE, EXP_TAG, FIRST_INITIALIZATION, JADE_TITLE +from jade.post.atlas_processor import AtlasProcessor from jade.post.excel_processor import ExcelProcessor from jade.post.raw_processor import RawProcessor from jade.run.benchmark import BenchmarkRunFactory @@ -42,7 +44,8 @@ def __init__(self, root: PathLike | None = None, skip_init: bool = False): # parse the config files self.run_cfg = RunConfig.from_root(self.tree) - # TODO read here also the post-processing config + # parse the post-processing config + self.pp_cfg = PostProcessConfig(self.tree.cfg.bench_pp) # Compute the global status self.status = GlobalStatus( @@ -173,27 +176,65 @@ def raw_process(self): def post_process(self): """Post-process the data.""" logging.info("Post-processing data") - # recover the post-processing benchmark configurations - bench_cfg = PostProcessConfig(self.tree.cfg.bench_pp) # load the pp code-lib requests with open(self.tree.cfg.pp_cfg) as f: - pp_cfg = yaml.safe_load(f) - for benchmark, options in pp_cfg.items(): - if options["process"]: - # prepare the new paths - pp_path = self.tree.get_new_post_bench_path(benchmark) - excel_folder = Path(pp_path, "excel") - os.mkdir(excel_folder) - # perform the excel processing - excel_cfg = bench_cfg.excel_cfgs[benchmark] - options["codelibs"] - processor = ExcelProcessor( - self.tree.raw, - excel_folder, - excel_cfg, - options["codelibs"], + to_pp = yaml.safe_load(f) + codelibs_tags = to_pp["code_libs"] + benchmarks = to_pp["benchmarks"] + + for benchmark in benchmarks: + logging.info(f"Post-processing {benchmark}") + # get the benchmark configurations + excel_cfg = self.pp_cfg.excel_cfgs[benchmark] + atlas_cfg = self.pp_cfg.atlas_cfgs[benchmark] + + code_libs = [] + # if exp is in the libraries, put it always first + if EXP_TAG in codelibs_tags: + code_libs.remove(EXP_TAG) + code_libs.insert(0, EXP_TAG) + + for codelib in code_libs: + # Check if the code-lib is available in this benchmark + # if yes, append it to the list + code, lib = get_code_lib(codelib) + if self.status.is_raw_available(codelib, benchmark): + code_libs.append((code, lib)) + else: + logging.info(f"{codelib} is not available for {benchmark}") + + # in case there are less than two code-libs skip the comparison + if len(code_libs) < 2: + logging.warning( + f"Less than two code-libs available for {benchmark}, skipped" ) - processor.process() + continue + + # prepare the new paths + pp_path = self.tree.get_new_post_bench_path(benchmark) + excel_folder = Path(pp_path, "excel") + atlas_folder = Path(pp_path, "atlas") + os.mkdir(excel_folder) + os.mkdir(atlas_folder) + + # perform the excel processing + excel_processor = ExcelProcessor( + self.tree.raw, + excel_folder, + excel_cfg, + code_libs, + ) + excel_processor.process() + + # perform the atlas processing + atlas_processor = AtlasProcessor( + self.tree.raw, + atlas_folder, + atlas_cfg, + code_libs, + files(resources).joinpath("atlas_template.docx"), + ) + atlas_processor.process() def start_config_gui(self): """Start the configuration GUI.""" diff --git a/src/jade/config/pp_config.py b/src/jade/config/pp_config.py index 17d4a6bb..91751750 100644 --- a/src/jade/config/pp_config.py +++ b/src/jade/config/pp_config.py @@ -3,6 +3,7 @@ import os from pathlib import Path +from jade.config.atlas_config import ConfigAtlasProcessor from jade.config.excel_config import ConfigExcelProcessor from jade.helper.aux_functions import PathLike @@ -16,4 +17,10 @@ def __init__(self, root_cfg_pp: PathLike): cfg = ConfigExcelProcessor.from_yaml(Path(root_cfg_pp, file)) excel_cfgs[cfg.benchmark] = cfg self.excel_cfgs = excel_cfgs - # TODO get all available config atlas processors + # Get all available config atlas processors + atlas_cfgs = {} + for file in os.listdir(Path(root_cfg_pp, "atlas")): + if file.endswith(".yaml") or file.endswith(".yml"): + cfg = ConfigAtlasProcessor.from_yaml(Path(root_cfg_pp, file)) + atlas_cfgs[cfg.benchmark] = cfg + self.atlas_cfgs = atlas_cfgs diff --git a/src/jade/config/status.py b/src/jade/config/status.py index 46f5ef70..9c8132ec 100644 --- a/src/jade/config/status.py +++ b/src/jade/config/status.py @@ -9,6 +9,7 @@ CODE_CHECKERS, PathLike, get_code_lib, + print_code_lib, ) from jade.helper.constants import CODE @@ -124,7 +125,7 @@ def was_simulated(self, code: CODE, lib: str, benchmark: str) -> bool: Returns ------- bool - Treu if the simulation was performed and successful, False otherwise. + True if the simulation was performed and successful, False otherwise. """ try: result = self.simulations[(code, lib, benchmark)] @@ -149,6 +150,86 @@ def get_successful_simulations( return successful_simulations + def get_all_raw(self) -> tuple[set[str], set[str]]: + """Get all the code-libraries and benchmarks for which the + at least one raw data is available. + + Returns + ------- + set[str] + list of code-libraries for which the raw data was processed. + set[str] + list of benchmarks for which the raw data was processed. + """ + code_libs = [] + benchmarks = [] + for code, lib, benchmark in self.raw_data.keys(): + code_libs.append(print_code_lib(code, lib)) + benchmarks.append(benchmark) + return set(code_libs), set(benchmarks) + + def get_codelibs_from_raw_benchmark(self, benchmarks: str | list[str]) -> set[str]: + """Get the list of codelib for which the raw data of the requested benchmark + is available. + + Parameters + ---------- + benchmark : str | list[str] + benchmark name. + + Returns + ------- + set[str] + list of codelib for which the raw data of the requested benchmark is available. + """ + if isinstance(benchmarks, str): + benchmarks = [benchmarks] + + codelibs = [] + for code, lib, bench in self.raw_data.keys(): + if bench in benchmarks: + codelibs.append(print_code_lib(code, lib)) + return set(codelibs) + + def get_benchmark_from_raw_codelib(self, codelibs: str | list[str]) -> set[str]: + """Get the list of benchmarks for which the raw data of the requested codelib + is available. + + Parameters + ---------- + codelib : str | list[str] + codelib name. + + Returns + ------- + set[str] + list of benchmarks for which the raw data of the requested codelib is available. + """ + benchmarks = [] + for code, lib, bench in self.raw_data.keys(): + if print_code_lib(code, lib) in codelibs: + benchmarks.append(bench) + return set(benchmarks) + + def is_raw_available(self, codelib: str, benchmark: str) -> bool: + """Check if the raw data is available for the given codelib and benchmark. + + Parameters + ---------- + codelib : str + codelib string. + benchmark : str + benchmark name. + + Returns + ------- + bool + True if the raw data is available, False otherwise. + """ + if codelib in self.get_codelibs_from_raw_benchmark(benchmark): + return True + return False + @dataclass class CodeLibRunStatus: diff --git a/src/jade/gui/post_config_gui.py b/src/jade/gui/post_config_gui.py new file mode 100644 index 00000000..f90ae575 --- /dev/null +++ b/src/jade/gui/post_config_gui.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import tkinter as tk +from tkinter import filedialog, messagebox, ttk + +import yaml + +from jade.config.status import GlobalStatus +from jade.helper.aux_functions import VerboseSafeDumper + + +class PostConfigGUI(tk.Tk): + def __init__(self, status: GlobalStatus): + super().__init__() + self.title("Post-Processing Configuration") + self.geometry("600x600") + self.status = status + + self.create_widgets() + + def create_widgets(self): + self.benchmark_label = ttk.Label(self, text="Select Benchmarks:") + self.benchmark_label.pack(pady=5) + + self.benchmark_listbox = tk.Listbox(self, selectmode=tk.MULTIPLE) + self.benchmark_listbox.pack(pady=5, fill=tk.BOTH, expand=True) + + self.code_lib_label = ttk.Label(self, text="Select Code Libraries:") + self.code_lib_label.pack(pady=5) + + self.code_lib_listbox = tk.Listbox(self, selectmode=tk.MULTIPLE) + self.code_lib_listbox.pack(pady=5, fill=tk.BOTH, expand=True) + + button_frame = ttk.Frame(self) + button_frame.pack(pady=10) + + self.update_button = ttk.Button( + button_frame, text="Update Selections", command=self.update_selections + ) + self.update_button.pack(side=tk.LEFT, padx=5) + + self.reset_button = ttk.Button( + button_frame, text="Reset Selections", command=self._reset_selections + ) + self.reset_button.pack(side=tk.LEFT, padx=5) + + self.save_button = ttk.Button( + button_frame, + text="Save Selections", + command=self._save_selections, + style="Green.TButton", + ) + self.save_button.pack(side=tk.LEFT, padx=5) + + style = ttk.Style() + style.configure("Green.TButton", foreground="green") + + self._reset_selections() + + def _save_selections(self): + selected_benchmarks = list(self.displayed_benchmarks) + selected_code_libs = list(self.displayed_code_libs) + + data = { + "benchmarks": selected_benchmarks, + "code_libs": selected_code_libs, + } + + file_path = filedialog.asksaveasfilename( + defaultextension=".yaml", + filetypes=[("YAML files", "*.yml"), ("All files", "*.*")], + ) + + if file_path: + with open(file_path, "w") as file: + yaml.dump( + data, file, default_flow_style=False, Dumper=VerboseSafeDumper + ) + messagebox.showinfo("Success", "Settings saved successfully!") + + def _reset_selections(self): + libs, benchmarks = status.get_all_raw() + self._init_list(self.benchmark_listbox, benchmarks) + self._init_list(self.code_lib_listbox, libs) + self.displayed_benchmarks = benchmarks + self.displayed_code_libs = libs + + @staticmethod + def _init_list(listbox: tk.Listbox, items: list | set): + # clear the listbox first + listbox.delete(0, tk.END) + for item in items: + listbox.insert(tk.END, item) + + def update_selections(self): + selected_benchmarks = [ + self.benchmark_listbox.get(i) for i in self.benchmark_listbox.curselection() + ] + selected_code_libs = [ + self.code_lib_listbox.get(i) for i in self.code_lib_listbox.curselection() + ] + + if selected_benchmarks: + # Get the available code libraries for the selected benchmarks + available_code_libs = status.get_codelibs_from_raw_benchmark( + selected_benchmarks + ) + self.code_lib_listbox.delete(0, tk.END) + for code_lib in available_code_libs: + if code_lib in self.displayed_code_libs: + self.code_lib_listbox.insert(tk.END, code_lib) + + # Only display the selected benchmarks + self.benchmark_listbox.delete(0, tk.END) + for benchmark in selected_benchmarks: + self.benchmark_listbox.insert(tk.END, benchmark) + + self.displayed_benchmarks = selected_benchmarks + self.displayed_code_libs = available_code_libs + + if selected_code_libs: + # Get the available benchmarks for the selected code libraries + available_benchmarks = status.get_benchmark_from_raw_codelib( + selected_code_libs + ) + self.benchmark_listbox.delete(0, tk.END) + for benchmark in available_benchmarks: + if benchmark in self.displayed_benchmarks: + self.benchmark_listbox.insert(tk.END, benchmark) + + # Only display the selected code libraries + self.code_lib_listbox.delete(0, tk.END) + for code_lib in selected_code_libs: + self.code_lib_listbox.insert(tk.END, code_lib) + + self.displayed_benchmarks = available_benchmarks + self.displayed_code_libs = selected_code_libs + + +if __name__ == "__main__": + status = GlobalStatus( + r"D:\DATA\laghida\Documents\GitHub\JADE\Code\tests\dummy_structure\simulations", + r"D:\DATA\laghida\Documents\GitHub\JADE\Code\tests\dummy_structure\raw_data", + ) + app = PostConfigGUI(status) + app.mainloop() diff --git a/src/jade/gui/config_gui.py b/src/jade/gui/run_config_gui.py similarity index 100% rename from src/jade/gui/config_gui.py rename to src/jade/gui/run_config_gui.py diff --git a/src/jade/helper/constants.py b/src/jade/helper/constants.py index 66f6cdc1..94725993 100644 --- a/src/jade/helper/constants.py +++ b/src/jade/helper/constants.py @@ -9,6 +9,8 @@ class CODE(Enum): EXPERIMENT = "exp" +EXP_TAG = "_exp_-_exp_" # tag for the experiment folder + # get all possile code tags in a list CODE_TAGS = [code.value for code in CODE] diff --git a/src/jade/resources/default_cfg/pp_cfg.yml b/src/jade/resources/default_cfg/pp_cfg.yml new file mode 100644 index 00000000..678a8f7a --- /dev/null +++ b/src/jade/resources/default_cfg/pp_cfg.yml @@ -0,0 +1,5 @@ +benchmarks: +- Sphere +code_libs: +- _mcnp_-_FENDL 3.2c_ +- _mcnp_-_ENDFB-VIII.0_ diff --git a/tests/config/test_status.py b/tests/config/test_status.py index 114c7e86..1f42ddf8 100644 --- a/tests/config/test_status.py +++ b/tests/config/test_status.py @@ -22,3 +22,19 @@ def test_was_simulated(self): assert status.was_simulated(CODE("mcnp"), "FENDL 3.2c", "Oktavian") assert not status.was_simulated(CODE("openmc"), "FENDL 3.2c", "Oktavian") assert not status.was_simulated(CODE("mcnp"), "FENDL 3.2c", "Sphere") + + def test_get_all_raw(self): + status = GlobalStatus(DUMMY_SIMULATIONS, DUMMY_RAW_RESULTS) + codelibs, benchmarks = status.get_all_raw() + assert len(codelibs) == 3 + assert len(benchmarks) == 2 + + def test_get_codelibs_from_raw_benchmark(self): + status = GlobalStatus(DUMMY_SIMULATIONS, DUMMY_RAW_RESULTS) + codelibs = status.get_codelibs_from_raw_benchmark("Oktavian") + assert len(codelibs) == 3 + + def test_get_benchmark_from_raw_codelib(self): + status = GlobalStatus(DUMMY_SIMULATIONS, DUMMY_RAW_RESULTS) + benchmarks = status.get_benchmark_from_raw_codelib("mcnp - FENDL 3.2c") + assert len(benchmarks) == 2 diff --git a/tests/gui/test_config_gui.py b/tests/gui/test_config_gui.py index f7b12eb2..60492b5e 100644 --- a/tests/gui/test_config_gui.py +++ b/tests/gui/test_config_gui.py @@ -8,7 +8,7 @@ from ttkthemes import ThemedTk import jade.resources -from jade.gui.config_gui import ConfigGUI +from jade.gui.run_config_gui import ConfigGUI DEFAULT_CFG = files(jade.resources).joinpath("default_cfg")