From a64adac930ae21869146514bcb10f0b7e30d2714 Mon Sep 17 00:00:00 2001 From: swenzel Date: Thu, 9 Jan 2025 13:12:56 +0100 Subject: [PATCH] O2DPG-MC: Possibility to pass additional external user settings This commit provides the possibility to inject external choices for configurable params into the MC workflow construction. This is done by setting up a JSON file, e.g. local_config.json, an by passing this file to the workflow construction with the new `--overwrite-config local_config.json` command line option. This new option allows to conveniently set configurable params from the outside (without having to specify the global list of configurations). The motivation for this development came from the wish to set the busy time for EMC digitization. This is now possible with the following local_config.json: ``` { "EMCSimParam": { "mBusyTime": 0 } } ``` Overall, this development scales to all such external settings and avoids adding specific detector-specific command line options in o2dpg_sim_workflow.py. --- MC/bin/o2dpg_sim_workflow.py | 16 ++++++++++++---- MC/bin/o2dpg_workflow_utils.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 12cf7b0da..9f0011889 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -35,7 +35,7 @@ sys.path.append(join(dirname(__file__), '.', 'o2dpg_workflow_utils')) -from o2dpg_workflow_utils import createTask, createGlobalInitTask, dump_workflow, adjust_RECO_environment, isActive, activate_detector, deactivate_detector, compute_n_workers +from o2dpg_workflow_utils import createTask, createGlobalInitTask, dump_workflow, adjust_RECO_environment, isActive, activate_detector, deactivate_detector, compute_n_workers, merge_dicts from o2dpg_qc_finalization_workflow import include_all_QC_finalization from o2dpg_sim_config import create_sim_config, create_geant_config, constructConfigKeyArg @@ -52,8 +52,9 @@ parser.add_argument('--conditionDB',help="CCDB url for QC workflows", default='http://alice-ccdb.cern.ch') parser.add_argument('--qcdbHost',help="QCDB url for QC object uploading", default='http://ali-qcdbmc-gpn.cern.ch:8083') parser.add_argument('--condition-not-after', type=int, help="only consider CCDB objects not created after this timestamp (for TimeMachine)", default=3385078236000) -parser.add_argument('--orbitsPerTF', type=int, help="Timeframe size in number of LHC orbits", default=128) -parser.add_argument('--anchor-config',help="JSON file to contextualise workflow with external configs (config values etc.) for instance comping from data reco workflows.", default='') +parser.add_argument('--orbitsPerTF', type=int, help="Timeframe size in number of LHC orbits", default=32) +parser.add_argument('--anchor-config',help="JSON file to contextualise workflow with external configs (config values etc.) for instance coming from data reco workflows.", default='') +parser.add_argument('--overwrite-config',help="extra JSON file with configs (config values etc.) overwriting defaults or the config coming from --anchor-config", default='') parser.add_argument('--dump-config',help="Dump JSON file with all settings used in workflow", default='user_config.json') parser.add_argument('-ns',type=int,help='number of signal events / timeframe', default=20) parser.add_argument('-gen',help='generator: pythia8, extgen', default='') @@ -199,6 +200,13 @@ def load_external_config(configfile): # we load a generic config print ("** Using generic config **") anchorConfig = create_sim_config(args) +# we apply additional external user choices for the configuration +# this will overwrite config from earlier stages +if args.overwrite_config != '': + # apply final JSON overwrite + config_overwrite = load_external_config(args.overwrite_config) + # merge the dictionaries into anchorConfig, the latter takes precedence + merge_dicts(anchorConfig, config_overwrite) # write this config config_key_param_path = args.dump_config @@ -1073,7 +1081,7 @@ def createRestDigiTask(name, det='ALLSMALLER'): FT0FV0EMCCTPDIGItask['cmd'] = ('','ln -nfs ../bkg_HitsFT0.root . ; ln -nfs ../bkg_HitsFV0.root . ; ln -nfs ../bkg_HitsEMC.root; ln -nfs ../bkg_Kine.root; ')[doembedding] FT0FV0EMCCTPDIGItask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + ' --onlyDet FT0,FV0,EMC,CTP --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) \ - + ' --disable-write-ini' + putConfigValuesNew(localCF={"DigiParams.seed" : str(TFSEED)}) \ + + ' --disable-write-ini' + putConfigValuesNew(listOfMainKeys=['EMCSimParam'], localCF={"DigiParams.seed" : str(TFSEED)}) \ + (' --combine-devices','')[args.no_combine_dpl_devices] + ('',' --disable-mc')[args.no_mc_labels] + QEDdigiargs \ + ' --forceSelectedDets' workflow['stages'].append(FT0FV0EMCCTPDIGItask) diff --git a/MC/bin/o2dpg_workflow_utils.py b/MC/bin/o2dpg_workflow_utils.py index b28333674..a196a9c86 100755 --- a/MC/bin/o2dpg_workflow_utils.py +++ b/MC/bin/o2dpg_workflow_utils.py @@ -308,3 +308,15 @@ def matches_or_inherits_label(taskid, label, cache): if (matches_or_inherits_label(taskid, "RECO", matches_label)): # now we do the final adjust (as annotation) in the workflow itself workflowspec['stages'][taskid]["alternative_alienv_package"] = package + +def merge_dicts(dict1, dict2): + """ + merges dict2 into dict1 (potentially overwriting values) + """ + for key, value in dict2.items(): + if key in dict1 and isinstance(dict1[key], dict) and isinstance(value, dict): + # If both are dictionaries, merge them recursively + merge_dicts(dict1[key], value) + else: + # Otherwise, overwrite dict1's value with dict2's value + dict1[key] = value