From fd40b8f3715f533cca81c4a66b4437b1019f8ea9 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 21 Aug 2023 19:26:42 -0400 Subject: [PATCH 01/20] trial the click method --- setup.py | 1 + src/swell/prepare.py | 0 src/swell/swell.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 src/swell/prepare.py create mode 100644 src/swell/swell.py diff --git a/setup.py b/setup.py index df7173aa..487d9756 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ include_package_data=True, entry_points={ 'console_scripts': [ + 'swell = swell.swell:main', 'swell_task = swell.tasks.base.task_base:main', 'swell_create_experiment = swell.deployment.bin.swell_create_experiment:main', 'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main', diff --git a/src/swell/prepare.py b/src/swell/prepare.py new file mode 100644 index 00000000..e69de29b diff --git a/src/swell/swell.py b/src/swell/swell.py new file mode 100644 index 00000000..686e8bce --- /dev/null +++ b/src/swell/swell.py @@ -0,0 +1,28 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +import click + +# -------------------------------------------------------------------------------------------------- + +@click.group() +def cli(): + pass + +@cli.command() +def prepare(): + click.echo("Preparing...") + +@cli.command() +def create(): + click.echo("Creating...") + + +def main(): + cli() + From 578b278e047156ead02323b292fb55da6c136bdf Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 23 Aug 2023 18:14:30 -0400 Subject: [PATCH 02/20] add single entry point --- .../deployment/bin/swell_create_experiment.py | 9 --- .../deployment/bin/swell_launch_experiment.py | 10 --- src/swell/prepare.py | 3 + src/swell/swell.py | 63 ++++++++++++++++--- .../convert_ncdiags-stable_build.yaml | 1 + src/swell/utilities/welcome_message.py | 3 +- 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/src/swell/deployment/bin/swell_create_experiment.py b/src/swell/deployment/bin/swell_create_experiment.py index 4a1ff6ba..db7bc3f2 100644 --- a/src/swell/deployment/bin/swell_create_experiment.py +++ b/src/swell/deployment/bin/swell_create_experiment.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -115,10 +113,3 @@ def main(config_file): # -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/bin/swell_launch_experiment.py b/src/swell/deployment/bin/swell_launch_experiment.py index 6e75825a..a32fe120 100644 --- a/src/swell/deployment/bin/swell_launch_experiment.py +++ b/src/swell/deployment/bin/swell_launch_experiment.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -132,7 +130,6 @@ def main(suite_path, workflow_manager, no_detach, log_path): deploy_workflow.logger.info('Launching workflow defined by files in \'' + suite_path + '\'.', False) deploy_workflow.logger.info('Experiment name: ' + experiment_name) - deploy_workflow.logger.info('Workflow manager: ' + workflow_manager) # Launch the workflow # ------------------- @@ -141,10 +138,3 @@ def main(suite_path, workflow_manager, no_detach, log_path): # -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/prepare.py b/src/swell/prepare.py index e69de29b..4005a0f2 100644 --- a/src/swell/prepare.py +++ b/src/swell/prepare.py @@ -0,0 +1,3 @@ +def prepare(): + print('This is a prepare function') + diff --git a/src/swell/swell.py b/src/swell/swell.py index 686e8bce..65a7d805 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -8,21 +8,68 @@ import click +from swell.deployment.prep_config import prepare_config +from swell.utilities.welcome_message import write_welcome_message + # -------------------------------------------------------------------------------------------------- +# Driver +# ------ @click.group() -def cli(): +def swell_driver(): pass -@cli.command() -def prepare(): - click.echo("Preparing...") +# Create experiment +# ----------------- +@swell_driver.command() +@click.option('-m', '--input_method', 'input_method', default='defaults', help='Method by which ' + + 'to create the YAML configuration file. Valid choices: \'defaults\', \'cli\'.') +@click.option('-p', '--platform', 'platform', default='nccs_discover', help='If using defaults ' + + 'for input_method this option is used to determine which platform to use for ' + + 'platform specific defaults.') +@click.option('-o', '--override', 'override', default=None, help='After generating the config ' + + 'file parameters inside can be overridden using value from the override config file.') +@click.option('-t', '--test', 'test', default=None, help='Override defaults with a preprepared ' + + 'test config file.') +def create_experiment(config): + + # Prepare the experiment config file + prepare_config(input_method, suite, platform, override, test) + + + click.echo("Running") + +# Launch experiment +# ------------------ +@swell_driver.command() +@click.option('-p', '--suite_path', 'suite_path', default=None, + help='Directory containing the suite file needed by the workflow manager') +@click.option('-b', '--no-detach', 'no_detach', is_flag=True, default=False, + help='Tells workflow manager to block until complete') +@click.option('-l', '--log_path', 'log_path', default=None, + help='Directory to receive workflow manager logging output') +def launch_experiment(): + click.echo("Launching...") -@cli.command() -def create(): - click.echo("Creating...") +# Tasks +# ----- +@swell_driver.command() +@click.argument('taskname') +def task(taskname): + click.echo("Task ") + print(taskname) +# Utilities +# --------- +@swell_driver.command() +@click.argument('utility') +def utility(utility): + click.echo("Utility ") + print(utility) +# Main +# ---- def main(): - cli() + write_welcome_message() + swell_driver() diff --git a/src/swell/test/suite_tests/convert_ncdiags-stable_build.yaml b/src/swell/test/suite_tests/convert_ncdiags-stable_build.yaml index 6960f773..1fadbfab 100644 --- a/src/swell/test/suite_tests/convert_ncdiags-stable_build.yaml +++ b/src/swell/test/suite_tests/convert_ncdiags-stable_build.yaml @@ -8,6 +8,7 @@ models: geos_atmosphere: cycle_times: - T00 + - T06 clean_patterns: - gsi_bcs/*.nc4 - gsi_bcs/*.txt diff --git a/src/swell/utilities/welcome_message.py b/src/swell/utilities/welcome_message.py index 2dfc5593..3b0c1a93 100644 --- a/src/swell/utilities/welcome_message.py +++ b/src/swell/utilities/welcome_message.py @@ -14,7 +14,7 @@ # -------------------------------------------------------------------------------------------------- -def write_welcome_message(application): +def write_welcome_message(): logger = Logger('') @@ -24,5 +24,4 @@ def write_welcome_message(application): logger.blank(f"\__ \\\ V V / __/ | | Version {__version__}", False) # noqa logger.blank(f"|___/ \_/\_/ \___|_|_| \x1B[4m\x1B[34mhttps://geos-esm.github.io/swell\033[0m", False) # noqa logger.blank(f" ", False) # noqa - logger.blank(f"Executing swell \'{application.lower()}\' application", False) # noqa logger.blank('', False) # noqa From dc2c165df993bbfafcf44770e4ddf66f41c20354 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 23 Aug 2023 18:20:37 -0400 Subject: [PATCH 03/20] add full interface for tasks --- src/swell/swell.py | 11 +++++++---- src/swell/tasks/base/task_base.py | 21 --------------------- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/src/swell/swell.py b/src/swell/swell.py index 65a7d805..7830954c 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -9,6 +9,7 @@ import click from swell.deployment.prep_config import prepare_config +from swell.tasks.base.task_base import task_main from swell.utilities.welcome_message import write_welcome_message # -------------------------------------------------------------------------------------------------- @@ -54,10 +55,12 @@ def launch_experiment(): # Tasks # ----- @swell_driver.command() -@click.argument('taskname') -def task(taskname): - click.echo("Task ") - print(taskname) +@click.argument('task') +@click.argument('config') +@click.option('-d', '--datetime', 'datetime', default=None) +@click.option('-m', '--model', 'model', default=None) +def task(task, config, datetime, model): + task_main(task, config, datetime, model) # Utilities # --------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index e1d1c039..8a0397c0 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -11,7 +11,6 @@ # standard imports from abc import ABC, abstractmethod -import click import importlib import os import time @@ -262,23 +261,3 @@ def task_main(task, config, datetime, model): # -------------------------------------------------------------------------------------------------- - - -@click.command() -@click.argument('task') -@click.argument('config') -@click.option('-d', '--datetime', 'datetime', default=None) -@click.option('-m', '--model', 'model', default=None) -def main(task, config, datetime, model): - - task_main(task, config, datetime, model) - - -# -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() - - -# -------------------------------------------------------------------------------------------------- From 678557d0c7ba9e64a6fb0968b2edd5c73ea04ce8 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 24 Aug 2023 14:03:04 -0500 Subject: [PATCH 04/20] rename suite tests --- src/swell/deployment/bin/__init__.py | 9 - .../deployment/bin/swell_prepare_config.py | 54 ------ ...ate_experiment.py => create_experiment.py} | 0 ...nch_experiment.py => launch_experiment.py} | 0 src/swell/deployment/prep_exp_dirs.py | 155 ----------------- src/swell/deployment/prep_suite.py | 164 ------------------ src/swell/prepare.py | 3 - ...table_build-ocean-5deg.yaml => 3dvar.yaml} | 0 ...build-ocean-5deg.yaml => 3dvar_cycle.yaml} | 0 ...stable_build.yaml => convert_ncdiags.yaml} | 0 ...s-stable_build.yaml => forecast_geos.yaml} | 0 ...osadas-stable_build.yaml => geosadas.yaml} | 0 .../{hofx-stable_build.yaml => hofx.yaml} | 0 ...ing-stable_build.yaml => ufo_testing.yaml} | 0 .../bin/swell_sat_db_processing.py | 0 15 files changed, 385 deletions(-) delete mode 100644 src/swell/deployment/bin/__init__.py delete mode 100644 src/swell/deployment/bin/swell_prepare_config.py rename src/swell/deployment/{bin/swell_create_experiment.py => create_experiment.py} (100%) rename src/swell/deployment/{bin/swell_launch_experiment.py => launch_experiment.py} (100%) delete mode 100644 src/swell/deployment/prep_exp_dirs.py delete mode 100644 src/swell/deployment/prep_suite.py delete mode 100644 src/swell/prepare.py rename src/swell/test/suite_tests/{3dvar-stable_build-ocean-5deg.yaml => 3dvar.yaml} (100%) rename src/swell/test/suite_tests/{3dvar_cycle-stable_build-ocean-5deg.yaml => 3dvar_cycle.yaml} (100%) rename src/swell/test/suite_tests/{convert_ncdiags-stable_build.yaml => convert_ncdiags.yaml} (100%) rename src/swell/test/suite_tests/{forecast_geos-stable_build.yaml => forecast_geos.yaml} (100%) rename src/swell/test/suite_tests/{geosadas-stable_build.yaml => geosadas.yaml} (100%) rename src/swell/test/suite_tests/{hofx-stable_build.yaml => hofx.yaml} (100%) rename src/swell/test/suite_tests/{ufo_testing-stable_build.yaml => ufo_testing.yaml} (100%) rename src/swell/{deployment => utilities}/bin/swell_sat_db_processing.py (100%) diff --git a/src/swell/deployment/bin/__init__.py b/src/swell/deployment/bin/__init__.py deleted file mode 100644 index f82722ea..00000000 --- a/src/swell/deployment/bin/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -import os - -repo_directory = os.path.dirname(__file__) diff --git a/src/swell/deployment/bin/swell_prepare_config.py b/src/swell/deployment/bin/swell_prepare_config.py deleted file mode 100644 index 9e831fd8..00000000 --- a/src/swell/deployment/bin/swell_prepare_config.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import click - -from swell.deployment.prep_config import prepare_config -from swell.utilities.welcome_message import write_welcome_message - - -# -------------------------------------------------------------------------------------------------- - - -@click.command() -@click.option('-m', '--input_method', 'input_method', default='defaults', help='Method by which ' + - 'to create the YAML configuration file. Valid choices: \'defaults\', \'cli\'.') -@click.option('-p', '--platform', 'platform', default='nccs_discover', help='If using defaults ' + - 'for input_method this option is used to determine which platform to use for ' + - 'platform specific defaults.') -@click.option('-o', '--override', 'override', default=None, help='After generating the config ' + - 'file parameters inside can be overridden using value from the override config file.') -@click.option('-t', '--test', 'test', default=None, help='Override defaults with a preprepared ' + - 'test config file.') -@click.argument('suite') -def main(input_method, suite, platform, override, test): - """ - SUITE argument determines which set of tasks are going to be run. - """ - - # Welcome message - # --------------- - write_welcome_message('Prepare Config') - - # Create suites object - # -------------------- - prepare_config(input_method, suite, platform, override, test) - - -# -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/bin/swell_create_experiment.py b/src/swell/deployment/create_experiment.py similarity index 100% rename from src/swell/deployment/bin/swell_create_experiment.py rename to src/swell/deployment/create_experiment.py diff --git a/src/swell/deployment/bin/swell_launch_experiment.py b/src/swell/deployment/launch_experiment.py similarity index 100% rename from src/swell/deployment/bin/swell_launch_experiment.py rename to src/swell/deployment/launch_experiment.py diff --git a/src/swell/deployment/prep_exp_dirs.py b/src/swell/deployment/prep_exp_dirs.py deleted file mode 100644 index d45863c2..00000000 --- a/src/swell/deployment/prep_exp_dirs.py +++ /dev/null @@ -1,155 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import copy -import os -import shutil - -from swell.swell_path import get_swell_path -from swell.utilities.jinja2 import template_string_jinja2 - - -# -------------------------------------------------------------------------------------------------- - - -def copy_eva_files(logger, swell_suite_path, exp_suite_path): - - # Repo eva files - eva_directory = os.path.join(swell_suite_path, 'eva') - - # Destination for eva files - destination_directory = os.path.join(exp_suite_path, 'eva') - - # If destination directory exists, delete it - if os.path.exists(destination_directory): - shutil.rmtree(destination_directory) - - # Copy all the files - shutil.copytree(eva_directory, destination_directory) - - -# -------------------------------------------------------------------------------------------------- - - -def copy_platform_files(logger, exp_suite_path, platform=None): - - # Copy platform related files to the suite directory - # -------------------------------------------------- - if platform is not None: - swell_lib_path = get_swell_path() - platform_path = os.path.join(swell_lib_path, 'deployment', 'platforms', platform) - - for s in ['modules', 'r2d2_config.yaml']: - src_file = os.path.split(s)[1] - src_path_file = os.path.join(platform_path, os.path.split(s)[0], src_file) - dst_path_file = os.path.join(exp_suite_path, '{}'.format(src_file)) - if os.path.exists(src_path_file): - logger.trace('Copying {} to {}'.format(src_path_file, dst_path_file)) - shutil.copy(src_path_file, dst_path_file) - - -# -------------------------------------------------------------------------------------------------- - - -def template_modules_file(logger, experiment_dict, exp_suite_path): - - # Modules file - # ------------ - modules_file = os.path.join(exp_suite_path, 'modules') - - # Only do if the suite needs modules - # ---------------------------------- - if os.path.exists(modules_file): - - # Swell bin path - # -------------- - swell_bin_path = shutil.which("swell_task") - swell_bin_path = os.path.split(swell_bin_path)[0] - - # Swell lib path - # -------------- - swell_lib_path = get_swell_path() - swell_lib_path = os.path.split(swell_lib_path)[0] - - # Swell suite path - # ---------------- - swell_sui_path = os.path.join(get_swell_path(), 'suites') - - # Dictionary of definitions - # ------------------------- - modules_dict = copy.copy(experiment_dict) - modules_dict['swell_bin_path'] = swell_bin_path - modules_dict['swell_lib_path'] = swell_lib_path - modules_dict['swell_sui_path'] = swell_sui_path - - # Open the file - # ------------- - with open(modules_file, 'r') as modules_file_open: - modules_file_str = modules_file_open.read() - - # Resolve templates - # ----------------- - modules_file_str = template_string_jinja2(logger, modules_file_str, modules_dict) - - # Overwrite the file - # ------------------ - with open(modules_file, 'w') as modules_file_open: - modules_file_open.write(modules_file_str) - - -# -------------------------------------------------------------------------------------------------- - - -def create_modules_csh(logger, exp_suite_path): - - # Modules file - # ------------ - modules_file = os.path.join(exp_suite_path, 'modules') - - # Only do if the suite needs modules - # ---------------------------------- - if os.path.exists(modules_file): - - # Open the file - # ------------- - with open(modules_file, 'r') as modules_file_open: - modules_file_lines = modules_file_open.readlines() - - # Replace some things - # ------------------- - for idx, modules_file_line in enumerate(modules_file_lines): - - # 'bash' to 'csh' - if 'bash' in modules_file_line: - modules_file_lines[idx] = modules_file_lines[idx].replace('bash', 'csh') - - # Export to setenv - if 'export' in modules_file_line: - modules_file_lines[idx] = modules_file_lines[idx].replace('export', 'setenv') - modules_file_lines[idx] = modules_file_lines[idx].replace('=', ' ') - - # Set PYTHONPATH - if 'PYTHONPATH=' in modules_file_line: - modules_file_lines[idx] = modules_file_lines[idx].replace('PYTHONPATH=', - 'setenv PYTHONPATH ') - - # Set path - if 'PATH=' in modules_file_line: - modules_file_lines[idx] = modules_file_lines[idx].replace('PATH=', 'set path = (') - modules_file_lines[idx] = modules_file_lines[idx].replace(':$PATH', ' $path)') - - # Overwrite the file - # ------------------ - with open(modules_file+'-csh', 'w') as modules_file_open: - for modules_file_line in modules_file_lines: - modules_file_open.write(modules_file_line) - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_suite.py b/src/swell/deployment/prep_suite.py deleted file mode 100644 index d053b195..00000000 --- a/src/swell/deployment/prep_suite.py +++ /dev/null @@ -1,164 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import os -import yaml - -from swell.utilities.dictionary import dict_get -from swell.utilities.jinja2 import template_string_jinja2 - - -# -------------------------------------------------------------------------------------------------- - - -def prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experiment_dict): - - # Open suite file from swell - # -------------------------- - with open(os.path.join(swell_suite_path, 'flow.cylc'), 'r') as file: - suite_file = file.read() - - # Copy the experiment dictionary to the rendering dictionary - # ---------------------------------------------------------- - render_dictionary = {} - - # Elements to copy from the experiment dictionary - # ----------------------------------------------- - render_elements = [ - 'start_cycle_point', - 'final_cycle_point', - 'runahead_limit', - 'model_components', - 'platform', - ] - - # Copy elements from experiment dictionary to render dictionary - # ------------------------------------------------------------- - for element in render_elements: - if element in experiment_dict: - render_dictionary[element] = experiment_dict[element] - - # Get unique list of cycle times with model flags to render dictionary - # -------------------------------------------------------------------- - - # Check if 'cycle_times' appears anywhere in the suite_file - if 'cycle_times' in suite_file: - - # Since cycle times are used, the render_dictionary will need to include cycle_times - model_components = dict_get(logger, experiment_dict, 'model_components', []) - - # If there are different model components then process each to gather cycle times - if len(model_components) > 0: - cycle_times = [] - for model_component in model_components: - cycle_times_mc = experiment_dict['models'][model_component]['cycle_times'] - cycle_times = list(set(cycle_times + cycle_times_mc)) - cycle_times.sort() - - cycle_times_dict_list = [] - for cycle_time in cycle_times: - cycle_time_dict = {} - cycle_time_dict['cycle_time'] = cycle_time - for model_component in model_components: - cycle_time_dict[model_component] = False - if cycle_time in experiment_dict['models'][model_component]['cycle_times']: - cycle_time_dict[model_component] = True - cycle_times_dict_list.append(cycle_time_dict) - - render_dictionary['cycle_times'] = cycle_times_dict_list - - # Otherwise check that experiment_dict has cycle_times - elif 'cycle_times' in experiment_dict: - - cycle_times = list(set(experiment_dict['cycle_times'])) - cycle_times.sort() - render_dictionary['cycle_times'] = cycle_times - - else: - - # Otherwise use logger to abort - logger.abort('The suite file required cycle_times but there are no model components ' + - 'to gather them from or they are not provided in the experiment ' + - 'dictionary.') - - # Look for a file called $HOME/.swell/slurm.yaml - # ---------------------------------------------- - yaml_path = os.path.expanduser("~/.swell/swell-slurm.yaml") - slurm_global = {} - if os.path.exists(yaml_path): - logger.info(f'Found file contianing swell slurm global values') - with open(yaml_path, "r") as yaml_file: - slurm_global = yaml.safe_load(yaml_file) - - # Set default values for global slurm values - account = 'g0613' - qos = 'allnccs' - partition = None - constraint = 'cas|sky' - - # Extract from slurm global file - if 'qos' in slurm_global: - qos = slurm_global['qos'] - - if 'partition' in slurm_global: - partition = slurm_global['partition'] - - if 'account' in slurm_global: - account = slurm_global['account'] - - if 'constraint' in slurm_global: - constraint = slurm_global['constraint'] - - # List of tasks using slurm - # ------------------------- - slurm_tasks = [ - 'BuildJedi', - 'BuildGeos', - 'GenerateBClimatology', - 'RunJediHofxExecutable', - 'RunJediVariationalExecutable', - 'RunJediUfoTestsExecutable', - 'RunGeosExecutable', - ] - - # Fill default values for slurm tasks - # ----------------------------------- - render_dictionary['scheduling'] = {} - for slurm_task in slurm_tasks: - render_dictionary['scheduling'][slurm_task] = {} - render_dictionary['scheduling'][slurm_task]['execution_time_limit'] = 'PT1H' - render_dictionary['scheduling'][slurm_task]['account'] = account - render_dictionary['scheduling'][slurm_task]['qos'] = qos - render_dictionary['scheduling'][slurm_task]['nodes'] = 1 - render_dictionary['scheduling'][slurm_task]['ntasks_per_node'] = 24 - render_dictionary['scheduling'][slurm_task]['constraint'] = constraint - render_dictionary['scheduling'][slurm_task]['partition'] = partition - - # Set some specific values for: - # ----------------------------- - - # run time - render_dictionary['scheduling']['BuildJedi']['execution_time_limit'] = 'PT3H' - render_dictionary['scheduling']['RunJediHofxExecutable']['execution_time_limit'] = 'PT2H' - - # nodes - render_dictionary['scheduling']['RunJediUfoTestsExecutable']['ntasks_per_node'] = 1 - - # Render the template - # ------------------- - new_suite_file = template_string_jinja2(logger, suite_file, render_dictionary) - - # Write suite file to experiment - # ------------------------------ - with open(os.path.join(exp_suite_path, 'flow.cylc'), 'w') as file: - file.write(new_suite_file) - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/prepare.py b/src/swell/prepare.py deleted file mode 100644 index 4005a0f2..00000000 --- a/src/swell/prepare.py +++ /dev/null @@ -1,3 +0,0 @@ -def prepare(): - print('This is a prepare function') - diff --git a/src/swell/test/suite_tests/3dvar-stable_build-ocean-5deg.yaml b/src/swell/test/suite_tests/3dvar.yaml similarity index 100% rename from src/swell/test/suite_tests/3dvar-stable_build-ocean-5deg.yaml rename to src/swell/test/suite_tests/3dvar.yaml diff --git a/src/swell/test/suite_tests/3dvar_cycle-stable_build-ocean-5deg.yaml b/src/swell/test/suite_tests/3dvar_cycle.yaml similarity index 100% rename from src/swell/test/suite_tests/3dvar_cycle-stable_build-ocean-5deg.yaml rename to src/swell/test/suite_tests/3dvar_cycle.yaml diff --git a/src/swell/test/suite_tests/convert_ncdiags-stable_build.yaml b/src/swell/test/suite_tests/convert_ncdiags.yaml similarity index 100% rename from src/swell/test/suite_tests/convert_ncdiags-stable_build.yaml rename to src/swell/test/suite_tests/convert_ncdiags.yaml diff --git a/src/swell/test/suite_tests/forecast_geos-stable_build.yaml b/src/swell/test/suite_tests/forecast_geos.yaml similarity index 100% rename from src/swell/test/suite_tests/forecast_geos-stable_build.yaml rename to src/swell/test/suite_tests/forecast_geos.yaml diff --git a/src/swell/test/suite_tests/geosadas-stable_build.yaml b/src/swell/test/suite_tests/geosadas.yaml similarity index 100% rename from src/swell/test/suite_tests/geosadas-stable_build.yaml rename to src/swell/test/suite_tests/geosadas.yaml diff --git a/src/swell/test/suite_tests/hofx-stable_build.yaml b/src/swell/test/suite_tests/hofx.yaml similarity index 100% rename from src/swell/test/suite_tests/hofx-stable_build.yaml rename to src/swell/test/suite_tests/hofx.yaml diff --git a/src/swell/test/suite_tests/ufo_testing-stable_build.yaml b/src/swell/test/suite_tests/ufo_testing.yaml similarity index 100% rename from src/swell/test/suite_tests/ufo_testing-stable_build.yaml rename to src/swell/test/suite_tests/ufo_testing.yaml diff --git a/src/swell/deployment/bin/swell_sat_db_processing.py b/src/swell/utilities/bin/swell_sat_db_processing.py similarity index 100% rename from src/swell/deployment/bin/swell_sat_db_processing.py rename to src/swell/utilities/bin/swell_sat_db_processing.py From 8293a461cf021cc8a191d9e00e46d3c0e5e72c0b Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 2 Oct 2023 22:11:05 -0400 Subject: [PATCH 05/20] commit latest work --- src/swell/tasks/base/task_registry.py | 44 ------------------- .../{3dvar.yaml => 3dvar-default.yaml} | 0 ...ar_cycle.yaml => 3dvar_cycle-default.yaml} | 0 ...iags.yaml => convert_ncdiags-default.yaml} | 0 ...t_geos.yaml => forecast_geos-default.yaml} | 0 .../{geosadas.yaml => geosadas-default.yaml} | 0 .../{hofx.yaml => hofx-default.yaml} | 0 ..._testing.yaml => ufo_testing-default.yaml} | 0 .../utilities/{bin => scripts}/__init__.py | 0 .../check_jedi_interface_templates.py | 0 .../swell_sat_db_processing.py | 0 .../{bin => scripts}/task_question_dicts.py | 0 .../task_question_dicts_defaults.py | 0 13 files changed, 44 deletions(-) delete mode 100644 src/swell/tasks/base/task_registry.py rename src/swell/test/suite_tests/{3dvar.yaml => 3dvar-default.yaml} (100%) rename src/swell/test/suite_tests/{3dvar_cycle.yaml => 3dvar_cycle-default.yaml} (100%) rename src/swell/test/suite_tests/{convert_ncdiags.yaml => convert_ncdiags-default.yaml} (100%) rename src/swell/test/suite_tests/{forecast_geos.yaml => forecast_geos-default.yaml} (100%) rename src/swell/test/suite_tests/{geosadas.yaml => geosadas-default.yaml} (100%) rename src/swell/test/suite_tests/{hofx.yaml => hofx-default.yaml} (100%) rename src/swell/test/suite_tests/{ufo_testing.yaml => ufo_testing-default.yaml} (100%) rename src/swell/utilities/{bin => scripts}/__init__.py (100%) rename src/swell/utilities/{bin => scripts}/check_jedi_interface_templates.py (100%) rename src/swell/utilities/{bin => scripts}/swell_sat_db_processing.py (100%) rename src/swell/utilities/{bin => scripts}/task_question_dicts.py (100%) rename src/swell/utilities/{bin => scripts}/task_question_dicts_defaults.py (100%) diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py deleted file mode 100644 index ff5a6219..00000000 --- a/src/swell/tasks/base/task_registry.py +++ /dev/null @@ -1,44 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -valid_tasks = [ - 'BuildGeosByLinking', - 'BuildGeos', - 'BuildJediByLinking', - 'BuildJedi', - 'CleanCycle', - 'CloneGeos', - 'CloneJedi', - 'EvaJediLog', - 'EvaObservations', - 'GenerateBClimatologyByLinking', - 'GenerateBClimatology', - 'GetBackgroundGeosExperiment', - 'GetBackground', - 'GetGeosAdasBackground', - 'GetGeosRestart', - 'GetGeovals', - 'GetGsiBc', - 'GetGsiNcdiag', - 'GetObservations', - 'GsiBcToIoda', - 'GsiNcdiagToIoda', - 'LinkGeosOutput', - 'MoveDaRestart', - 'MoveForecastRestart', - 'ObsProcessSetup', - 'PrepareAnalysis', - 'PrepGeosRunDir', - 'RemoveForecastDir', - 'RunGeosExecutable', - 'RunJediHofxExecutable', - 'RunJediUfoTestsExecutable', - 'RunJediVariationalExecutable', - 'SaveObsDiags', - 'SaveRestart', - 'StageJedi', - 'StoreBackground', -] diff --git a/src/swell/test/suite_tests/3dvar.yaml b/src/swell/test/suite_tests/3dvar-default.yaml similarity index 100% rename from src/swell/test/suite_tests/3dvar.yaml rename to src/swell/test/suite_tests/3dvar-default.yaml diff --git a/src/swell/test/suite_tests/3dvar_cycle.yaml b/src/swell/test/suite_tests/3dvar_cycle-default.yaml similarity index 100% rename from src/swell/test/suite_tests/3dvar_cycle.yaml rename to src/swell/test/suite_tests/3dvar_cycle-default.yaml diff --git a/src/swell/test/suite_tests/convert_ncdiags.yaml b/src/swell/test/suite_tests/convert_ncdiags-default.yaml similarity index 100% rename from src/swell/test/suite_tests/convert_ncdiags.yaml rename to src/swell/test/suite_tests/convert_ncdiags-default.yaml diff --git a/src/swell/test/suite_tests/forecast_geos.yaml b/src/swell/test/suite_tests/forecast_geos-default.yaml similarity index 100% rename from src/swell/test/suite_tests/forecast_geos.yaml rename to src/swell/test/suite_tests/forecast_geos-default.yaml diff --git a/src/swell/test/suite_tests/geosadas.yaml b/src/swell/test/suite_tests/geosadas-default.yaml similarity index 100% rename from src/swell/test/suite_tests/geosadas.yaml rename to src/swell/test/suite_tests/geosadas-default.yaml diff --git a/src/swell/test/suite_tests/hofx.yaml b/src/swell/test/suite_tests/hofx-default.yaml similarity index 100% rename from src/swell/test/suite_tests/hofx.yaml rename to src/swell/test/suite_tests/hofx-default.yaml diff --git a/src/swell/test/suite_tests/ufo_testing.yaml b/src/swell/test/suite_tests/ufo_testing-default.yaml similarity index 100% rename from src/swell/test/suite_tests/ufo_testing.yaml rename to src/swell/test/suite_tests/ufo_testing-default.yaml diff --git a/src/swell/utilities/bin/__init__.py b/src/swell/utilities/scripts/__init__.py similarity index 100% rename from src/swell/utilities/bin/__init__.py rename to src/swell/utilities/scripts/__init__.py diff --git a/src/swell/utilities/bin/check_jedi_interface_templates.py b/src/swell/utilities/scripts/check_jedi_interface_templates.py similarity index 100% rename from src/swell/utilities/bin/check_jedi_interface_templates.py rename to src/swell/utilities/scripts/check_jedi_interface_templates.py diff --git a/src/swell/utilities/bin/swell_sat_db_processing.py b/src/swell/utilities/scripts/swell_sat_db_processing.py similarity index 100% rename from src/swell/utilities/bin/swell_sat_db_processing.py rename to src/swell/utilities/scripts/swell_sat_db_processing.py diff --git a/src/swell/utilities/bin/task_question_dicts.py b/src/swell/utilities/scripts/task_question_dicts.py similarity index 100% rename from src/swell/utilities/bin/task_question_dicts.py rename to src/swell/utilities/scripts/task_question_dicts.py diff --git a/src/swell/utilities/bin/task_question_dicts_defaults.py b/src/swell/utilities/scripts/task_question_dicts_defaults.py similarity index 100% rename from src/swell/utilities/bin/task_question_dicts_defaults.py rename to src/swell/utilities/scripts/task_question_dicts_defaults.py From 8924fb8c25bf72bf1dd2739a2ef8abb42c0838a4 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 2 Oct 2023 22:11:32 -0400 Subject: [PATCH 06/20] commit latest work --- src/swell/deployment/create_experiment.py | 319 ++++++++++++++++-- src/swell/deployment/launch_experiment.py | 15 +- .../platforms/generic/suite_questions.yaml | 2 +- src/swell/deployment/prep_config.py | 21 +- src/swell/deployment/prep_config_base.py | 1 + src/swell/deployment/prep_config_utils.py | 29 ++ src/swell/swell.py | 161 +++++++-- src/swell/tasks/base/task_base.py | 36 +- src/swell/test/code_tests/code_tests.py | 2 +- src/swell/test/test_driver.py | 29 ++ .../scripts/check_jedi_interface_templates.py | 9 - .../scripts/swell_sat_db_processing.py | 4 - .../utilities/scripts/task_question_dicts.py | 9 - .../scripts/task_question_dicts_defaults.py | 9 - src/swell/utilities/scripts/utility_driver.py | 51 +++ src/swell/utilities/suite_utils.py | 50 +++ 16 files changed, 618 insertions(+), 129 deletions(-) create mode 100644 src/swell/deployment/prep_config_utils.py create mode 100644 src/swell/test/test_driver.py create mode 100644 src/swell/utilities/scripts/utility_driver.py create mode 100644 src/swell/utilities/suite_utils.py diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index db7bc3f2..1d72c590 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -8,40 +8,29 @@ # -------------------------------------------------------------------------------------------------- -import click +import copy import os import shutil import yaml -from swell.deployment.prep_exp_dirs import copy_eva_files, copy_platform_files, \ - template_modules_file, create_modules_csh -from swell.deployment.prep_suite import prepare_cylc_suite_jinja2 from swell.swell_path import get_swell_path from swell.utilities.dictionary import dict_get from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.logger import Logger -from swell.utilities.welcome_message import write_welcome_message # -------------------------------------------------------------------------------------------------- -@click.command() -@click.argument('config_file') -def main(config_file): - - # Welcome message - # --------------- - write_welcome_message('Create Experiment') +def create_experiment_directory(experiment_dict_str): # Create a logger # --------------- logger = Logger('SwellCreateExperiment') - # Load experiment file - # -------------------- - with open(config_file, 'r') as ymlfile: - experiment_dict = yaml.safe_load(ymlfile) + # Load the string using yaml + # -------------------------- + experiment_dict = yaml.safe_load(experiment_dict_str) # Extract from the config # ----------------------- @@ -50,15 +39,23 @@ def main(config_file): platform = dict_get(logger, experiment_dict, 'platform', None) suite_to_run = dict_get(logger, experiment_dict, 'suite_to_run') + # Write out some info + # ------------------- + logger.info(f'Creating experiment: \'{experiment_id}\'') + logger.info(f'Experiment root: \'{experiment_root}\'') + # Make the suite directory # ------------------------ exp_path = os.path.join(experiment_root, experiment_id) exp_suite_path = os.path.join(exp_path, experiment_id+'-suite') + + print(exp_suite_path) os.makedirs(exp_suite_path, 0o755, exist_ok=True) - # Copy experiment file to suite dir - # --------------------------------- - shutil.copyfile(config_file, os.path.join(exp_suite_path, 'experiment.yaml')) + # Write dictionary (with comments) to YAML file + # --------------------------------------------- + with open(os.path.join(exp_suite_path, 'experiment.yaml'), 'w') as file: + file.write(experiment_dict_str) # Copy suite and platform files to experiment suite directory # ----------------------------------------------------------- @@ -108,8 +105,290 @@ def main(config_file): # ---------------------------------------- logger.info(' ') logger.info(' Experiment successfully installed. To launch experiment use: ') - logger.info(' swell_launch_experiment --suite_path ' + exp_suite_path, False) + logger.info(' swell launch-experiment ' + exp_suite_path, False) logger.info(' ') # -------------------------------------------------------------------------------------------------- + + +def copy_eva_files(logger, swell_suite_path, exp_suite_path): + + # Repo eva files + eva_directory = os.path.join(swell_suite_path, 'eva') + + # Destination for eva files + destination_directory = os.path.join(exp_suite_path, 'eva') + + # If destination directory exists, delete it + if os.path.exists(destination_directory): + shutil.rmtree(destination_directory) + + # Copy all the files + shutil.copytree(eva_directory, destination_directory) + + +# -------------------------------------------------------------------------------------------------- + + +def copy_platform_files(logger, exp_suite_path, platform=None): + + # Copy platform related files to the suite directory + # -------------------------------------------------- + if platform is not None: + swell_lib_path = get_swell_path() + platform_path = os.path.join(swell_lib_path, 'deployment', 'platforms', platform) + + for s in ['modules', 'r2d2_config.yaml']: + src_file = os.path.split(s)[1] + src_path_file = os.path.join(platform_path, os.path.split(s)[0], src_file) + dst_path_file = os.path.join(exp_suite_path, '{}'.format(src_file)) + if os.path.exists(src_path_file): + logger.trace('Copying {} to {}'.format(src_path_file, dst_path_file)) + shutil.copy(src_path_file, dst_path_file) + + +# -------------------------------------------------------------------------------------------------- + + +def template_modules_file(logger, experiment_dict, exp_suite_path): + + # Modules file + # ------------ + modules_file = os.path.join(exp_suite_path, 'modules') + + # Only do if the suite needs modules + # ---------------------------------- + if os.path.exists(modules_file): + + # Swell bin path + # -------------- + swell_bin_path = shutil.which("swell_task") + swell_bin_path = os.path.split(swell_bin_path)[0] + + # Swell lib path + # -------------- + swell_lib_path = get_swell_path() + swell_lib_path = os.path.split(swell_lib_path)[0] + + # Swell suite path + # ---------------- + swell_sui_path = os.path.join(get_swell_path(), 'suites') + + # Dictionary of definitions + # ------------------------- + modules_dict = copy.copy(experiment_dict) + modules_dict['swell_bin_path'] = swell_bin_path + modules_dict['swell_lib_path'] = swell_lib_path + modules_dict['swell_sui_path'] = swell_sui_path + + # Open the file + # ------------- + with open(modules_file, 'r') as modules_file_open: + modules_file_str = modules_file_open.read() + + # Resolve templates + # ----------------- + modules_file_str = template_string_jinja2(logger, modules_file_str, modules_dict) + + # Overwrite the file + # ------------------ + with open(modules_file, 'w') as modules_file_open: + modules_file_open.write(modules_file_str) + + +# -------------------------------------------------------------------------------------------------- + + +def create_modules_csh(logger, exp_suite_path): + + # Modules file + # ------------ + modules_file = os.path.join(exp_suite_path, 'modules') + + # Only do if the suite needs modules + # ---------------------------------- + if os.path.exists(modules_file): + + # Open the file + # ------------- + with open(modules_file, 'r') as modules_file_open: + modules_file_lines = modules_file_open.readlines() + + # Replace some things + # ------------------- + for idx, modules_file_line in enumerate(modules_file_lines): + + # 'bash' to 'csh' + if 'bash' in modules_file_line: + modules_file_lines[idx] = modules_file_lines[idx].replace('bash', 'csh') + + # Export to setenv + if 'export' in modules_file_line: + modules_file_lines[idx] = modules_file_lines[idx].replace('export', 'setenv') + modules_file_lines[idx] = modules_file_lines[idx].replace('=', ' ') + + # Set PYTHONPATH + if 'PYTHONPATH=' in modules_file_line: + modules_file_lines[idx] = modules_file_lines[idx].replace('PYTHONPATH=', + 'setenv PYTHONPATH ') + + # Set path + if 'PATH=' in modules_file_line: + modules_file_lines[idx] = modules_file_lines[idx].replace('PATH=', 'set path = (') + modules_file_lines[idx] = modules_file_lines[idx].replace(':$PATH', ' $path)') + + # Overwrite the file + # ------------------ + with open(modules_file+'-csh', 'w') as modules_file_open: + for modules_file_line in modules_file_lines: + modules_file_open.write(modules_file_line) + + +# -------------------------------------------------------------------------------------------------- + + +def prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experiment_dict): + + # Open suite file from swell + # -------------------------- + with open(os.path.join(swell_suite_path, 'flow.cylc'), 'r') as file: + suite_file = file.read() + + # Copy the experiment dictionary to the rendering dictionary + # ---------------------------------------------------------- + render_dictionary = {} + + # Elements to copy from the experiment dictionary + # ----------------------------------------------- + render_elements = [ + 'start_cycle_point', + 'final_cycle_point', + 'runahead_limit', + 'model_components', + 'platform', + ] + + # Copy elements from experiment dictionary to render dictionary + # ------------------------------------------------------------- + for element in render_elements: + if element in experiment_dict: + render_dictionary[element] = experiment_dict[element] + + # Get unique list of cycle times with model flags to render dictionary + # -------------------------------------------------------------------- + + # Check if 'cycle_times' appears anywhere in the suite_file + if 'cycle_times' in suite_file: + + # Since cycle times are used, the render_dictionary will need to include cycle_times + model_components = dict_get(logger, experiment_dict, 'model_components', []) + + # If there are different model components then process each to gather cycle times + if len(model_components) > 0: + cycle_times = [] + for model_component in model_components: + cycle_times_mc = experiment_dict['models'][model_component]['cycle_times'] + cycle_times = list(set(cycle_times + cycle_times_mc)) + cycle_times.sort() + + cycle_times_dict_list = [] + for cycle_time in cycle_times: + cycle_time_dict = {} + cycle_time_dict['cycle_time'] = cycle_time + for model_component in model_components: + cycle_time_dict[model_component] = False + if cycle_time in experiment_dict['models'][model_component]['cycle_times']: + cycle_time_dict[model_component] = True + cycle_times_dict_list.append(cycle_time_dict) + + render_dictionary['cycle_times'] = cycle_times_dict_list + + # Otherwise check that experiment_dict has cycle_times + elif 'cycle_times' in experiment_dict: + + cycle_times = list(set(experiment_dict['cycle_times'])) + cycle_times.sort() + render_dictionary['cycle_times'] = cycle_times + + else: + + # Otherwise use logger to abort + logger.abort('The suite file required cycle_times but there are no model components ' + + 'to gather them from or they are not provided in the experiment ' + + 'dictionary.') + + # Look for a file called $HOME/.swell/slurm.yaml + # ---------------------------------------------- + yaml_path = os.path.expanduser("~/.swell/swell-slurm.yaml") + slurm_global = {} + if os.path.exists(yaml_path): + logger.info(f'Found file contianing swell slurm global values') + with open(yaml_path, "r") as yaml_file: + slurm_global = yaml.safe_load(yaml_file) + + # Set default values for global slurm values + account = 'g0613' + qos = 'allnccs' + partition = None + constraint = 'cas|sky' + + # Extract from slurm global file + if 'qos' in slurm_global: + qos = slurm_global['qos'] + + if 'partition' in slurm_global: + partition = slurm_global['partition'] + + if 'account' in slurm_global: + account = slurm_global['account'] + + if 'constraint' in slurm_global: + constraint = slurm_global['constraint'] + + # List of tasks using slurm + # ------------------------- + slurm_tasks = [ + 'BuildJedi', + 'BuildGeos', + 'GenerateBClimatology', + 'RunJediHofxExecutable', + 'RunJediVariationalExecutable', + 'RunJediUfoTestsExecutable', + 'RunGeosExecutable', + ] + + # Fill default values for slurm tasks + # ----------------------------------- + render_dictionary['scheduling'] = {} + for slurm_task in slurm_tasks: + render_dictionary['scheduling'][slurm_task] = {} + render_dictionary['scheduling'][slurm_task]['execution_time_limit'] = 'PT1H' + render_dictionary['scheduling'][slurm_task]['account'] = account + render_dictionary['scheduling'][slurm_task]['qos'] = qos + render_dictionary['scheduling'][slurm_task]['nodes'] = 1 + render_dictionary['scheduling'][slurm_task]['ntasks_per_node'] = 24 + render_dictionary['scheduling'][slurm_task]['constraint'] = constraint + render_dictionary['scheduling'][slurm_task]['partition'] = partition + + # Set some specific values for: + # ----------------------------- + + # run time + render_dictionary['scheduling']['BuildJedi']['execution_time_limit'] = 'PT3H' + render_dictionary['scheduling']['RunJediHofxExecutable']['execution_time_limit'] = 'PT2H' + + # nodes + render_dictionary['scheduling']['RunJediUfoTestsExecutable']['ntasks_per_node'] = 1 + + # Render the template + # ------------------- + new_suite_file = template_string_jinja2(logger, suite_file, render_dictionary) + + # Write suite file to experiment + # ------------------------------ + with open(os.path.join(exp_suite_path, 'flow.cylc'), 'w') as file: + file.write(new_suite_file) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/launch_experiment.py b/src/swell/deployment/launch_experiment.py index a32fe120..64fe27cd 100644 --- a/src/swell/deployment/launch_experiment.py +++ b/src/swell/deployment/launch_experiment.py @@ -8,7 +8,6 @@ # standard imports -import click import os # local imports @@ -95,16 +94,7 @@ def cylc_run_experiment(self): # NB: Could be a factory based on workflow_manag # -------------------------------------------------------------------------------------------------- -@click.command() -@click.option('-p', '--suite_path', 'suite_path', default=None, - help='Directory containing the suite file needed by the workflow manager') -@click.option('-w', '--workflow_manager', 'workflow_manager', default='cylc', - help='Workflow manager to be used') -@click.option('-b', '--no-detach', 'no_detach', is_flag=True, default=False, - help='Tells workflow manager to block until complete') -@click.option('-l', '--log_path', 'log_path', default=None, - help='Directory to receive workflow manager logging output') -def main(suite_path, workflow_manager, no_detach, log_path): +def launch(suite_path, no_detach, log_path): # Welcome message # --------------- @@ -133,8 +123,7 @@ def main(suite_path, workflow_manager, no_detach, log_path): # Launch the workflow # ------------------- - if workflow_manager == 'cylc': - deploy_workflow.cylc_run_experiment() + deploy_workflow.cylc_run_experiment() # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/platforms/generic/suite_questions.yaml b/src/swell/deployment/platforms/generic/suite_questions.yaml index 2793cbf0..889ba481 100644 --- a/src/swell/deployment/platforms/generic/suite_questions.yaml +++ b/src/swell/deployment/platforms/generic/suite_questions.yaml @@ -2,7 +2,7 @@ experiment_id: default_value: swell-{{suite_to_run}} experiment_root: - default_value: /home/${USER}/SwellExperiments + default_value: $HOME/SwellExperiments r2d2_local_path: default_value: /home/${USER}/R2D2DataStore/Local diff --git a/src/swell/deployment/prep_config.py b/src/swell/deployment/prep_config.py index 6d408f3c..e8d4a317 100644 --- a/src/swell/deployment/prep_config.py +++ b/src/swell/deployment/prep_config.py @@ -43,7 +43,7 @@ def update_model_components(logger, experiment_dict, comment_dict): # -------------------------------------------------------------------------------------------------- -def prepare_config(method, suite, platform, override, test): +def prepare_config(suite, method, platform, override, test): # Create a logger # --------------- @@ -92,6 +92,11 @@ def prepare_config(method, suite, platform, override, test): experiment_dict_string = os.path.expandvars(experiment_dict_string) experiment_dict = yaml.safe_load(experiment_dict_string) + # If method is defaults then override with default suite test + # ----------------------------------------------------------- + if method == 'defaults': + test = 'default' + # Point to a particular pre-existing dictionary used for testing # -------------------------------------------------------------- if test is not None: @@ -145,21 +150,9 @@ def prepare_config(method, suite, platform, override, test): experiment_dict_string_comments = add_comments_to_dictionary(experiment_dict_string, comment_dict) - # Dictionary file to write - # ------------------------ - cwd = os.getcwd() - experiment_id = dict_get(logger, experiment_dict, 'experiment_id') - exp_dict_file = os.path.join(cwd, f'{experiment_id}.yaml') - - # Write dictionary to YAML file - # ----------------------------- - with open(exp_dict_file, 'w') as file: - file.write(experiment_dict_string_comments) - logger.info(f'Prepared configuration file written to {exp_dict_file}', False) - # Return path to dictionary file # ------------------------------ - return exp_dict_file + return experiment_dict_string_comments # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_config_base.py b/src/swell/deployment/prep_config_base.py index 53ee0bae..3edba11e 100644 --- a/src/swell/deployment/prep_config_base.py +++ b/src/swell/deployment/prep_config_base.py @@ -16,6 +16,7 @@ from swell.swell_path import get_swell_path + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_config_utils.py b/src/swell/deployment/prep_config_utils.py new file mode 100644 index 00000000..a12a0552 --- /dev/null +++ b/src/swell/deployment/prep_config_utils.py @@ -0,0 +1,29 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os + +from swell.swell_path import get_swell_path + + +# -------------------------------------------------------------------------------------------------- + + +def get_platforms(): + + # Path to platforms + platform_directory = os.path.join(get_swell_path(), 'deployment', 'platforms') + + # List all directories in platform_directory + return [dir for dir in os.listdir(platform_directory) + if os.path.isdir(os.path.join(platform_directory, dir))] + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 7830954c..cb2a46ed 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -4,75 +4,164 @@ # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + # -------------------------------------------------------------------------------------------------- + import click from swell.deployment.prep_config import prepare_config -from swell.tasks.base.task_base import task_main +from swell.deployment.prep_config_utils import get_platforms +from swell.deployment.create_experiment import create_experiment_directory +from swell.deployment.launch_experiment import launch +from swell.tasks.base.task_base import task_wrapper, get_tasks +from swell.test.test_driver import test_wrapper, valid_tests +from swell.utilities.suite_utils import get_suite_tests, get_suites from swell.utilities.welcome_message import write_welcome_message + # -------------------------------------------------------------------------------------------------- -# Driver -# ------ + @click.group() def swell_driver(): + """ + Swell Driver Group + + This is the main command group for swell. It serves as a container for various commands + related to experiment creation, launching, tasks, and utilities. + """ pass -# Create experiment -# ----------------- + +# -------------------------------------------------------------------------------------------------- + + @swell_driver.command() -@click.option('-m', '--input_method', 'input_method', default='defaults', help='Method by which ' + - 'to create the YAML configuration file. Valid choices: \'defaults\', \'cli\'.') -@click.option('-p', '--platform', 'platform', default='nccs_discover', help='If using defaults ' + - 'for input_method this option is used to determine which platform to use for ' + - 'platform specific defaults.') -@click.option('-o', '--override', 'override', default=None, help='After generating the config ' + - 'file parameters inside can be overridden using value from the override config file.') -@click.option('-t', '--test', 'test', default=None, help='Override defaults with a preprepared ' + - 'test config file.') -def create_experiment(config): +@click.argument('suite', type=click.Choice(get_suites())) +@click.option('-m', '--input_method', 'input_method', default='defaults', + type=click.Choice(['defaults', 'cli']), + help='Method by which to create the YAML configuration file. If choosing defaults ' + 'the setting for the default suite test will be used. If using CLI you will be ' + 'led through the questions to configure the experiment.') +@click.option('-p', '--platform', 'platform', type=click.Choice(get_platforms()), + help='If using defaults for input_method, this option is used ' + 'to determine which platform to use for platform specific defaults.') +@click.option('-o', '--override', 'override', default=None, + help='After generating the config file, parameters inside can be overridden ' + 'using values from the override config file.') +@click.option('-t', '--test', 'test', default=None, type=click.Choice(get_suite_tests()), + help='Specify a particular suite test to use for the configuration. This can still ' + 'be overridden.') +def create_experiment(suite, input_method, platform, override, test): + """ + Create a new experiment configuration and directory + + This command creates an experiment directory based on the provided suite name and options. + + Arguments: \n + suite (str): Name of the suite you wish to run. \n + + """ + experiment_dict_str = prepare_config(suite, input_method, platform, override, test) + create_experiment_directory(experiment_dict_str) - # Prepare the experiment config file - prepare_config(input_method, suite, platform, override, test) +# -------------------------------------------------------------------------------------------------- - click.echo("Running") -# Launch experiment -# ------------------ @swell_driver.command() -@click.option('-p', '--suite_path', 'suite_path', default=None, - help='Directory containing the suite file needed by the workflow manager') +@click.argument('suite_path') @click.option('-b', '--no-detach', 'no_detach', is_flag=True, default=False, - help='Tells workflow manager to block until complete') + help='Tells the workflow manager not to detach. That is to say run the entire ' + 'run the entire workflow in the foreground and pass back a return code.') @click.option('-l', '--log_path', 'log_path', default=None, - help='Directory to receive workflow manager logging output') -def launch_experiment(): - click.echo("Launching...") + help='Directory to receive workflow manager logging output (instead of ' + '$HOME/cylc-run/)') +def launch_experiment(suite_path, no_detach, log_path): + """ + Launch an experiment with the cylc workflow manager + + This command launches an experiment using the provided suite path and options. + + Arguments: \n + suite_path (str): Path to where the flow.cylc and associated suite files are located. \n + + """ + launch(suite_path, no_detach, log_path) + + +# -------------------------------------------------------------------------------------------------- + -# Tasks -# ----- @swell_driver.command() -@click.argument('task') +@click.argument('task', type=click.Choice(get_tasks())) @click.argument('config') @click.option('-d', '--datetime', 'datetime', default=None) @click.option('-m', '--model', 'model', default=None) def task(task, config, datetime, model): - task_main(task, config, datetime, model) + """ + Run a workflow task + + This command executes a task using the provided task name, configuration file and options. + + Arguments:\n + task (str): Name of the task to execute.\n + config (str): Path to the configuration file for the task.\n + + """ + task_wrapper(task, config, datetime, model) + + +# -------------------------------------------------------------------------------------------------- + -# Utilities -# --------- @swell_driver.command() @click.argument('utility') def utility(utility): - click.echo("Utility ") + """ + Run a utility script + + This command performs a utility operation specified by the utility argument. + + Arguments:\n + utility (str): Name of the utility operation to perform.\n + + """ print(utility) -# Main -# ---- + +# -------------------------------------------------------------------------------------------------- + + +@swell_driver.command() +@click.argument('test', type=click.Choice(valid_tests)) +def test(test): + """ + Run one of the test suites + + This command performs the test specified by the test argument. + + Arguments:\n + test (str): Name of the test to execute. + + """ + print(test) + test_wrapper(test) + + +# -------------------------------------------------------------------------------------------------- + + def main(): - write_welcome_message() + """ + Main Function + This function is the entry point for swell. It writes a welcome message and + sets up the driver group. + """ + write_welcome_message() swell_driver() + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 8a0397c0..27e54ba8 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -11,13 +11,14 @@ # standard imports from abc import ABC, abstractmethod +import glob import importlib import os import time # swell imports -from swell.tasks.base.task_registry import valid_tasks -from swell.utilities.case_switching import camel_case_to_snake_case +from swell.swell_path import get_swell_path +from swell.utilities.case_switching import camel_case_to_snake_case, snake_case_to_camel_case from swell.utilities.config import Config from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams from swell.utilities.datetime import Datetime @@ -223,19 +224,28 @@ def create_task(self, task, config, datetime, model): # -------------------------------------------------------------------------------------------------- +def get_tasks(): -def task_main(task, config, datetime, model): + # Path to tasks + tasks_directory = os.path.join(get_swell_path(), 'tasks', '*.py') - # For security check that task is in the registry - if task not in valid_tasks: - valid_task_logger = Logger('CheckValidTasks') - valid_task_logger.info(' ') - valid_task_logger.info('Task \'' + task + '\' not found in registry; valid tasks are:') - valid_tasks.sort() - for valid_task in valid_tasks: - valid_task_logger.info(' ' + valid_task) - valid_task_logger.info(' ') - valid_task_logger.abort('ABORT: Task not found in task registry.') + # List of tasks + task_files = sorted(glob.glob(tasks_directory)) + + # Get just the task name + tasks = [] + for task_file in task_files: + base_name = os.path.basename(task_file) + if '__' not in base_name: + tasks.append(snake_case_to_camel_case(base_name[0:-3])) + + # Return list of valid task choices + return tasks + +# -------------------------------------------------------------------------------------------------- + + +def task_wrapper(task, config, datetime, model): # Create the object constrc_start = time.perf_counter() diff --git a/src/swell/test/code_tests/code_tests.py b/src/swell/test/code_tests/code_tests.py index 062dc05a..47bf5311 100644 --- a/src/swell/test/code_tests/code_tests.py +++ b/src/swell/test/code_tests/code_tests.py @@ -18,7 +18,7 @@ # -------------------------------------------------------------------------------------------------- -def main(): +def code_tests(): # Create a logger logger = Logger('TestSuite') diff --git a/src/swell/test/test_driver.py b/src/swell/test/test_driver.py new file mode 100644 index 00000000..a0f195be --- /dev/null +++ b/src/swell/test/test_driver.py @@ -0,0 +1,29 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import importlib + + +# -------------------------------------------------------------------------------------------------- + +valid_tests = ['code_tests'] + +# -------------------------------------------------------------------------------------------------- + +def test_wrapper(test): + + # Test script + test_script_file = 'swell.test.'+test+'.'+test + + # Import the correct method + test_method = getattr(importlib.import_module(test_script_file), test) + + # Run the test + test_method() diff --git a/src/swell/utilities/scripts/check_jedi_interface_templates.py b/src/swell/utilities/scripts/check_jedi_interface_templates.py index fa4ccf03..81902edd 100644 --- a/src/swell/utilities/scripts/check_jedi_interface_templates.py +++ b/src/swell/utilities/scripts/check_jedi_interface_templates.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -99,10 +97,3 @@ def main(): # -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/scripts/swell_sat_db_processing.py b/src/swell/utilities/scripts/swell_sat_db_processing.py index d1e17c18..62cafa8c 100644 --- a/src/swell/utilities/scripts/swell_sat_db_processing.py +++ b/src/swell/utilities/scripts/swell_sat_db_processing.py @@ -90,7 +90,3 @@ def main(config): # create yamls make_yamls(processed_data, yaml_out_dir) - - -if __name__ == '__main__': - main() diff --git a/src/swell/utilities/scripts/task_question_dicts.py b/src/swell/utilities/scripts/task_question_dicts.py index b2e94c8c..7691a31c 100644 --- a/src/swell/utilities/scripts/task_question_dicts.py +++ b/src/swell/utilities/scripts/task_question_dicts.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -162,10 +160,3 @@ def tq_dicts(): # -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - tq_dicts() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/scripts/task_question_dicts_defaults.py b/src/swell/utilities/scripts/task_question_dicts_defaults.py index 8c065263..6c4ee645 100644 --- a/src/swell/utilities/scripts/task_question_dicts_defaults.py +++ b/src/swell/utilities/scripts/task_question_dicts_defaults.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -248,10 +246,3 @@ def tq_dicts_defaults(): # -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - tq_dicts_defaults() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/scripts/utility_driver.py b/src/swell/utilities/scripts/utility_driver.py new file mode 100644 index 00000000..96f56bb1 --- /dev/null +++ b/src/swell/utilities/scripts/utility_driver.py @@ -0,0 +1,51 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os +import importlib + +from swell.swell_path import get_swell_path +from swell.utilities.case_switching import snake_case_to_camel_case +from swell.test.code_tests.code_tests import code_tests + + +# -------------------------------------------------------------------------------------------------- + +def get_utilities(): + + # Path to util scripts + util_scripts_dir = os.path.join(get_swell_path(), 'utilities', 'scripts', '*.py') + + # List of tasks + util_script_files = sorted(glob.glob(util_scripts_dir)) + + # Get just the task name + util_scripts = [] + for util_script_file in util_script_files: + base_name = os.path.basename(util_script_file) + if '__' not in base_name: + util_scripts.append(snake_case_to_camel_case(base_name[0:-3])) + + # Return list of valid task choices + return util_scripts + +# -------------------------------------------------------------------------------------------------- + +def utility_wrapper(utility): + + # Test script + test_script_file = 'swell.utility.scripts.'+utility+'.'+utility + + # Import the correct method + utility_method = getattr(importlib.import_module(test_script_file), utility) + + # Run the test + utility_method() diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py new file mode 100644 index 00000000..3ace88a5 --- /dev/null +++ b/src/swell/utilities/suite_utils.py @@ -0,0 +1,50 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os + +from swell.swell_path import get_swell_path + + +# -------------------------------------------------------------------------------------------------- + + +def get_suites(): + + # Path to platforms + suites_directory = os.path.join(get_swell_path(), 'suites') + + # List all directories in platform_directory + return [dir for dir in os.listdir(suites_directory) + if os.path.isdir(os.path.join(suites_directory, dir))] + + +# -------------------------------------------------------------------------------------------------- + + +def get_suite_tests(): + + # Path to platforms + suite_tests_directory = os.path.join(get_swell_path(), 'test', 'suite_tests', '*.yaml') + + # List of tasks + suite_test_files = sorted(glob.glob(suite_tests_directory)) + + # Get just the task name + suite_tests = [] + for suite_test_file in suite_test_files: + suite_tests.append(os.path.basename(suite_test_file)[0:-5]) + + # Return list of valid task choices + return suite_tests + + +# -------------------------------------------------------------------------------------------------- From 39ef4f6722a4a910ba4bd66f26ac4bd29cb541be Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 15:49:20 -0400 Subject: [PATCH 07/20] update reference to swell task in flows --- src/swell/deployment/create_experiment.py | 12 +- src/swell/deployment/launch_experiment.py | 6 +- src/swell/deployment/prep_config.py | 33 +++++- src/swell/deployment/prep_config_base.py | 26 +++-- src/swell/deployment/prep_config_utils.py | 9 +- src/swell/suites/3dvar/flow.cylc | 26 ++--- src/swell/suites/3dvar_cycle/flow.cylc | 46 ++++---- src/swell/suites/build_geos/flow.cylc | 6 +- src/swell/suites/build_jedi/flow.cylc | 6 +- src/swell/suites/convert_ncdiags/flow.cylc | 16 +-- src/swell/suites/forecast_geos/flow.cylc | 18 +-- src/swell/suites/geosadas/flow.cylc | 20 ++-- src/swell/suites/hofx/flow.cylc | 22 ++-- src/swell/suites/letkf/flow.cylc | 22 ++-- src/swell/suites/ufo_testing/flow.cylc | 22 ++-- src/swell/swell.py | 123 +++++++++++++++------ src/swell/utilities/suite_utils.py | 13 ++- 17 files changed, 263 insertions(+), 163 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 1d72c590..045e7e33 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -41,15 +41,13 @@ def create_experiment_directory(experiment_dict_str): # Write out some info # ------------------- - logger.info(f'Creating experiment: \'{experiment_id}\'') - logger.info(f'Experiment root: \'{experiment_root}\'') + logger.info(f'Creating experiment: \'{experiment_id}\' in \'{experiment_root}\'') # Make the suite directory # ------------------------ exp_path = os.path.join(experiment_root, experiment_id) exp_suite_path = os.path.join(exp_path, experiment_id+'-suite') - print(exp_suite_path) os.makedirs(exp_suite_path, 0o755, exist_ok=True) # Write dictionary (with comments) to YAML file @@ -104,9 +102,10 @@ def create_experiment_directory(experiment_dict_str): # Write out launch command for convenience # ---------------------------------------- logger.info(' ') - logger.info(' Experiment successfully installed. To launch experiment use: ') - logger.info(' swell launch-experiment ' + exp_suite_path, False) - logger.info(' ') + logger.info('Experiment successfully installed. To launch experiment use: ') + logger.info(' ', False) + logger.info(' swell launch ' + exp_suite_path, False) + logger.info(' ', False) # -------------------------------------------------------------------------------------------------- @@ -323,7 +322,6 @@ def prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experime yaml_path = os.path.expanduser("~/.swell/swell-slurm.yaml") slurm_global = {} if os.path.exists(yaml_path): - logger.info(f'Found file contianing swell slurm global values') with open(yaml_path, "r") as yaml_file: slurm_global = yaml.safe_load(yaml_file) diff --git a/src/swell/deployment/launch_experiment.py b/src/swell/deployment/launch_experiment.py index 64fe27cd..644b24de 100644 --- a/src/swell/deployment/launch_experiment.py +++ b/src/swell/deployment/launch_experiment.py @@ -94,11 +94,7 @@ def cylc_run_experiment(self): # NB: Could be a factory based on workflow_manag # -------------------------------------------------------------------------------------------------- -def launch(suite_path, no_detach, log_path): - - # Welcome message - # --------------- - write_welcome_message('Launch Experiment') +def launch_experiment(suite_path, no_detach, log_path): # Get the path to where the suite files are located # ------------------------------------------------- diff --git a/src/swell/deployment/prep_config.py b/src/swell/deployment/prep_config.py index d450c210..a5193da4 100644 --- a/src/swell/deployment/prep_config.py +++ b/src/swell/deployment/prep_config.py @@ -32,7 +32,6 @@ def update_model_components(logger, experiment_dict, comment_dict): # model_components_actual then remove it from model for model in model_components_actual: if model not in model_components_wanted: - logger.info(f'Removing model {model} from model_components') del (experiment_dict['models'][model]) # Loop over all elements of the comment dictionay and remove any redundant keys for key in list(comment_dict.keys()): @@ -43,6 +42,38 @@ def update_model_components(logger, experiment_dict, comment_dict): # -------------------------------------------------------------------------------------------------- +def clone_config(configuration, experiment_id, method, platform, advanced): + + # Create a logger + logger = Logger('SwellCloneExperiment') + + # Check that configuration exists and is a YAML file + if not os.path.isfile(configuration): + logger.abort(f'The provided configuration file {configuration} does not exist') + + # Open the target experiment YAML. It will be used as the override + with open(configuration, 'r') as f: + override_dict = yaml.safe_load(f) + + # Check that override_dict has a suite key and get the suite name + if 'suite_to_run' not in override_dict: + logger.abort('The provided configuration file does not have a \'suite_to_run\' key') + suite = override_dict['suite_to_run'] + + # The user may want to run on a different platform (if so adjust the override) + if platform is not None: + override_dict['platform'] = platform + + # Set the experiment_id in the override dictionary + override_dict['experiment_id'] = experiment_id + + # First create the configuration for the experiment. + return prepare_config(suite, method, override_dict['platform'], override_dict, advanced) + + +# -------------------------------------------------------------------------------------------------- + + def prepare_config(suite, method, platform, override, advanced): # Create a logger diff --git a/src/swell/deployment/prep_config_base.py b/src/swell/deployment/prep_config_base.py index 18ad6d0b..9e73f3c2 100644 --- a/src/swell/deployment/prep_config_base.py +++ b/src/swell/deployment/prep_config_base.py @@ -106,12 +106,18 @@ def __init__(self, logger, dictionary_file, suite, platform, override, advanced) # Open user selected override dictionary if override is not None: - logger.info(f'Overriding experiment dictionary settings using {override}') - - select_override_file = os.path.join(override) - - with open(select_override_file, 'r') as select_override_yml: - self.override_dictionary = yaml.safe_load(select_override_yml) + logger.info(f'Overriding experiment dictionary settings using override dictionary') + + # If override is a dictionary then use it directly + if isinstance(override, dict): + self.override_dictionary = override + elif isinstance(override, str): + # If override is a string then assume it is a path to a yaml file + with open(override, 'r') as select_override_yml: + self.override_dictionary = yaml.safe_load(select_override_yml) + else: + logger.abort(f'Override must be a dictionary or a path to a yaml file. ' + + f'Instead it is {type(override)}') self.override_models() @@ -123,8 +129,8 @@ def execute(self): # defaults gets nothing if self.prep_using == 'Cli': - print("Please answer the following questions to generate your experiment " + - "configuration YAML file.\n") + self.logger("Please answer the following questions to generate your experiment " + + "configuration YAML file.\n") # Set current dictionary variable which is needed for answer changes self.current_dictionary = {} @@ -270,10 +276,6 @@ def open_flow(self): continue base_task_list.append(task_name) - # The following lists of tasks added to logger - self.logger.info(f'Base tasks selected include the following: {base_task_list}. \n' + - f'Model tasks selected include the following: {model_task_list}.') - return base_task_list, model_task_list # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_config_utils.py b/src/swell/deployment/prep_config_utils.py index a12a0552..20c89f90 100644 --- a/src/swell/deployment/prep_config_utils.py +++ b/src/swell/deployment/prep_config_utils.py @@ -21,9 +21,14 @@ def get_platforms(): # Path to platforms platform_directory = os.path.join(get_swell_path(), 'deployment', 'platforms') + platforms = [dir for dir in os.listdir(platform_directory) + if os.path.isdir(os.path.join(platform_directory, dir))] + + # If anything in platforms contains '__' remove it from platforms list + platforms = [platform for platform in platforms if '__' not in platform] + # List all directories in platform_directory - return [dir for dir in os.listdir(platform_directory) - if os.path.isdir(os.path.join(platform_directory, dir))] + return platforms # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 1457de7c..4ef7efde 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -101,13 +101,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -123,22 +123,22 @@ {% for model_component in model_components %} [[StageJedi-{{model_component}}]] - script = "swell_task StageJedi $config -m {{model_component}}" + script = "swell task StageJedi $config -m {{model_component}}" [[StageJediCycle-{{model_component}}]] - script = "swell_task StageJedi $config -d $datetime -m {{model_component}}" + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" [[ GetBackground-{{model_component}} ]] - script = "swell_task GetBackground $config -d $datetime -m {{model_component}}" + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" [[GetObservations-{{model_component}}]] - script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" [[GenerateBClimatologyByLinking-{{model_component}}]] - script = "swell_task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" + script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell_task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -153,16 +153,16 @@ {% endif %} [[EvaJediLog-{{model_component}}]] - script = "swell_task EvaJediLog $config -d $datetime -m {{model_component}}" + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" [[EvaObservations-{{model_component}}]] - script = "swell_task EvaObservations $config -d $datetime -m {{model_component}}" + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" [[SaveObsDiags-{{model_component}}]] - script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" [[CleanCycle-{{model_component}}]] - script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index 4344968d..85024d48 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -133,13 +133,13 @@ # Tasks # ----- [[CloneGeos]] - script = "swell_task CloneGeos $config" + script = "swell task CloneGeos $config" [[BuildGeosByLinking]] - script = "swell_task BuildGeosByLinking $config" + script = "swell task BuildGeosByLinking $config" [[BuildGeos]] - script = "swell_task BuildGeos $config" + script = "swell task BuildGeos $config" platform = {{platform}} execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} [[[directives]]] @@ -153,13 +153,13 @@ {% endif %} [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -174,7 +174,7 @@ {% endif %} [[RunGeosExecutable]] - script = "swell_task RunGeosExecutable $config -d $datetime" + script = "swell task RunGeosExecutable $config -d $datetime" platform = {{platform}} execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -189,39 +189,39 @@ {% endif %} [[PrepGeosRunDir]] - script = "swell_task PrepGeosRunDir $config -d $datetime" + script = "swell task PrepGeosRunDir $config -d $datetime" [[RemoveForecastDir]] - script = "swell_task RemoveForecastDir $config -d $datetime" + script = "swell task RemoveForecastDir $config -d $datetime" [[GetGeosRestart]] - script = "swell_task GetGeosRestart $config -d $datetime" + script = "swell task GetGeosRestart $config -d $datetime" {% for model_component in model_components %} [[LinkGeosOutput-{{model_component}}]] - script = "swell_task LinkGeosOutput $config -d $datetime -m {{model_component}}" + script = "swell task LinkGeosOutput $config -d $datetime -m {{model_component}}" [[MoveDaRestart-{{model_component}}]] - script = "swell_task MoveDaRestart $config -d $datetime -m {{model_component}}" + script = "swell task MoveDaRestart $config -d $datetime -m {{model_component}}" [[SaveRestart-{{model_component}}]] - script = "swell_task SaveRestart $config -d $datetime -m {{model_component}}" + script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" [[StageJedi-{{model_component}}]] - script = "swell_task StageJedi $config -m {{model_component}}" + script = "swell task StageJedi $config -m {{model_component}}" [[StageJediCycle-{{model_component}}]] - script = "swell_task StageJedi $config -d $datetime -m {{model_component}}" + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" [[GetObservations-{{model_component}}]] - script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" [[GenerateBClimatologyByLinking-{{model_component}}]] - script = "swell_task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" + script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell_task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -236,19 +236,19 @@ {% endif %} [[EvaJediLog-{{model_component}}]] - script = "swell_task EvaJediLog $config -d $datetime -m {{model_component}}" + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" [[EvaObservations-{{model_component}}]] - script = "swell_task EvaObservations $config -d $datetime -m {{model_component}}" + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" [[SaveObsDiags-{{model_component}}]] - script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" [[PrepareAnalysis-{{model_component}}]] - script = "swell_task PrepareAnalysis $config -d $datetime -m {{model_component}}" + script = "swell task PrepareAnalysis $config -d $datetime -m {{model_component}}" [[CleanCycle-{{model_component}}]] - script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/flow.cylc b/src/swell/suites/build_geos/flow.cylc index 8923ed33..16455d48 100644 --- a/src/swell/suites/build_geos/flow.cylc +++ b/src/swell/suites/build_geos/flow.cylc @@ -39,13 +39,13 @@ # Tasks # ----- [[CloneGeos]] - script = "swell_task CloneGeos $config" + script = "swell task CloneGeos $config" [[BuildGeosByLinking]] - script = "swell_task BuildGeosByLinking $config" + script = "swell task BuildGeosByLinking $config" [[BuildGeos]] - script = "swell_task BuildGeos $config" + script = "swell task BuildGeos $config" platform = {{platform}} execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} [[[directives]]] diff --git a/src/swell/suites/build_jedi/flow.cylc b/src/swell/suites/build_jedi/flow.cylc index 13760ee8..2f673b79 100644 --- a/src/swell/suites/build_jedi/flow.cylc +++ b/src/swell/suites/build_jedi/flow.cylc @@ -39,13 +39,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] diff --git a/src/swell/suites/convert_ncdiags/flow.cylc b/src/swell/suites/convert_ncdiags/flow.cylc index 0f6fbd58..c437e4df 100644 --- a/src/swell/suites/convert_ncdiags/flow.cylc +++ b/src/swell/suites/convert_ncdiags/flow.cylc @@ -70,13 +70,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -91,18 +91,18 @@ {% endif %} [[ GetGsiBc ]] - script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" [[ GsiBcToIoda ]] - script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" [[ GetGsiNcdiag ]] - script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" [[ GsiNcdiagToIoda ]] - script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" [[CleanCycle]] - script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" + script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/flow.cylc b/src/swell/suites/forecast_geos/flow.cylc index 4bffa71e..8b40bd20 100644 --- a/src/swell/suites/forecast_geos/flow.cylc +++ b/src/swell/suites/forecast_geos/flow.cylc @@ -75,13 +75,13 @@ # Tasks # ----- [[CloneGeos]] - script = "swell_task CloneGeos $config" + script = "swell task CloneGeos $config" [[BuildGeosByLinking]] - script = "swell_task BuildGeosByLinking $config" + script = "swell task BuildGeosByLinking $config" [[BuildGeos]] - script = "swell_task BuildGeos $config" + script = "swell task BuildGeos $config" platform = {{platform}} execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} [[[directives]]] @@ -95,22 +95,22 @@ {% endif %} [[PrepGeosRunDir]] - script = "swell_task PrepGeosRunDir $config -d $datetime" + script = "swell task PrepGeosRunDir $config -d $datetime" [[RemoveForecastDir]] - script = "swell_task RemoveForecastDir $config -d $datetime" + script = "swell task RemoveForecastDir $config -d $datetime" [[GetGeosRestart]] - script = "swell_task GetGeosRestart $config -d $datetime" + script = "swell task GetGeosRestart $config -d $datetime" [[MoveForecastRestart]] - script = "swell_task MoveForecastRestart $config -d $datetime" + script = "swell task MoveForecastRestart $config -d $datetime" [[SaveRestart]] - script = "swell_task SaveRestart $config -d $datetime" + script = "swell task SaveRestart $config -d $datetime" [[RunGeosExecutable]] - script = "swell_task RunGeosExecutable $config -d $datetime" + script = "swell task RunGeosExecutable $config -d $datetime" platform = {{platform}} execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} [[[directives]]] diff --git a/src/swell/suites/geosadas/flow.cylc b/src/swell/suites/geosadas/flow.cylc index 888d91ec..f92cba9c 100644 --- a/src/swell/suites/geosadas/flow.cylc +++ b/src/swell/suites/geosadas/flow.cylc @@ -70,31 +70,31 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[StageJedi]] - script = "swell_task StageJedi $config -m geos_atmosphere" + script = "swell task StageJedi $config -m geos_atmosphere" [[ GetGsiBc ]] - script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" [[ GsiBcToIoda ]] - script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" [[ GetGsiNcdiag ]] - script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" [[ GsiNcdiagToIoda ]] - script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" [[ GetGeosAdasBackground ]] - script = "swell_task GetGeosAdasBackground $config -d $datetime -m geos_atmosphere" + script = "swell task GetGeosAdasBackground $config -d $datetime -m geos_atmosphere" [[RunJediVariationalExecutable]] - script = "swell_task RunJediVariationalExecutable $config -d $datetime -m geos_atmosphere" + script = "swell task RunJediVariationalExecutable $config -d $datetime -m geos_atmosphere" platform = {{platform}} execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -109,6 +109,6 @@ {% endif %} [[CleanCycle]] - script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" + script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index 7b09767d..529f5379 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -94,13 +94,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -116,19 +116,19 @@ {% for model_component in model_components %} [[StageJedi-{{model_component}}]] - script = "swell_task StageJedi $config -m {{model_component}}" + script = "swell task StageJedi $config -m {{model_component}}" [[StageJediCycle-{{model_component}}]] - script = "swell_task StageJedi $config -d $datetime -m {{model_component}}" + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" [[ GetBackground-{{model_component}} ]] - script = "swell_task GetBackground $config -d $datetime -m {{model_component}}" + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" [[GetObservations-{{model_component}}]] - script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" [[RunJediHofxExecutable-{{model_component}}]] - script = "swell_task RunJediHofxExecutable $config -d $datetime -m {{model_component}}" + script = "swell task RunJediHofxExecutable $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["RunJediHofxExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -143,13 +143,13 @@ {% endif %} [[EvaObservations-{{model_component}}]] - script = "swell_task EvaObservations $config -d $datetime -m {{model_component}}" + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" [[SaveObsDiags-{{model_component}}]] - script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" [[CleanCycle-{{model_component}}]] - script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/letkf/flow.cylc b/src/swell/suites/letkf/flow.cylc index 07ca8d10..2b787151 100644 --- a/src/swell/suites/letkf/flow.cylc +++ b/src/swell/suites/letkf/flow.cylc @@ -94,13 +94,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -116,19 +116,19 @@ {% for model_component in model_components %} [[StageJedi-{{model_component}}]] - script = "swell_task StageJedi $config -m {{model_component}}" + script = "swell task StageJedi $config -m {{model_component}}" [[StageJediCycle-{{model_component}}]] - script = "swell_task StageJedi $config -d $datetime -m {{model_component}}" + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" [[ GetEnsemble-{{model_component}} ]] - script = "swell_task GetEnsemble $config -d $datetime -m {{model_component}}" + script = "swell task GetEnsemble $config -d $datetime -m {{model_component}}" [[GetObservations-{{model_component}}]] - script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" [[RunJediLetkfExecutable-{{model_component}}]] - script = "swell_task RunJediLetkfExecutable $config -d $datetime -m {{model_component}}" + script = "swell task RunJediLetkfExecutable $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["RunJediLetkfExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -143,13 +143,13 @@ {% endif %} [[EvaObservations-{{model_component}}]] - script = "swell_task EvaObservations $config -d $datetime -m {{model_component}}" + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" [[SaveObsDiags-{{model_component}}]] - script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" [[CleanCycle-{{model_component}}]] - script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index fd047820..4f93c18a 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -81,13 +81,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -102,22 +102,22 @@ {% endif %} [[ GetGsiBc ]] - script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" [[ GsiBcToIoda ]] - script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" [[ GetGsiNcdiag ]] - script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" [[ GsiNcdiagToIoda ]] - script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" [[ GetGeovals ]] - script = "swell_task GetGeovals $config -d $datetime -m geos_atmosphere" + script = "swell task GetGeovals $config -d $datetime -m geos_atmosphere" [[RunJediUfoTestsExecutable]] - script = "swell_task RunJediUfoTestsExecutable $config -d $datetime -m geos_atmosphere" + script = "swell task RunJediUfoTestsExecutable $config -d $datetime -m geos_atmosphere" platform = {{platform}} execution time limit = {{scheduling["RunJediUfoTestsExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -132,9 +132,9 @@ {% endif %} [[EvaObservations]] - script = "swell_task EvaObservations $config -d $datetime -m geos_atmosphere" + script = "swell task EvaObservations $config -d $datetime -m geos_atmosphere" [[CleanCycle]] - script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" + script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index cb2a46ed..813723cc 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -10,13 +10,13 @@ import click -from swell.deployment.prep_config import prepare_config +from swell.deployment.prep_config import clone_config, prepare_config from swell.deployment.prep_config_utils import get_platforms from swell.deployment.create_experiment import create_experiment_directory -from swell.deployment.launch_experiment import launch +from swell.deployment.launch_experiment import launch_experiment from swell.tasks.base.task_base import task_wrapper, get_tasks from swell.test.test_driver import test_wrapper, valid_tests -from swell.utilities.suite_utils import get_suite_tests, get_suites +from swell.utilities.suite_utils import get_suites from swell.utilities.welcome_message import write_welcome_message @@ -26,36 +26,67 @@ @click.group() def swell_driver(): """ - Swell Driver Group + Welcome to swell! - This is the main command group for swell. It serves as a container for various commands + This is the top level driver for swell. It serves as a container for various commands related to experiment creation, launching, tasks, and utilities. + + The normal process for createing and running an experiment is to issue: + + swell create + + followed by + + swell launch + """ pass +# -------------------------------------------------------------------------------------------------- + +# Help strings for optional arguments + +input_method_help = 'Method by which to create the YAML configuration file. If choosing ' + \ + 'defaults the setting for the default suite test will be used. If using ' + \ + 'CLI you will be led through the questions to configure the experiment.' + +platform_help = 'If using defaults for input_method, this option is used to determine which ' + \ + 'platform to use for platform specific defaults. Options are ' + \ + str(get_platforms()) + +override_help = 'After generating the config file, parameters inside can be overridden ' + \ + 'using values from the override config file.' + +advanced_help = 'Show configuration questions which are otherwise not shown to the user.' + +no_detach_help = 'Tells the workflow manager not to detach. That is to say run the entire ' + \ + 'run the entire workflow in the foreground and pass back a return code.' + +log_path_help = 'Directory to receive workflow manager logging output (instead of ' + \ + '$HOME/cylc-run/)' + +datetime_help = 'Datetime to use for task execution. Format is yyyy-mm-ddThh:mm:ss. Note that ' + \ + 'non-numeric characters will be stripped from the string. Minutes and seconds ' + \ + 'are optional.' + +model_help = 'Data assimilation system. I.e. the model being initialized by data assimilation.' + + # -------------------------------------------------------------------------------------------------- @swell_driver.command() @click.argument('suite', type=click.Choice(get_suites())) @click.option('-m', '--input_method', 'input_method', default='defaults', - type=click.Choice(['defaults', 'cli']), - help='Method by which to create the YAML configuration file. If choosing defaults ' - 'the setting for the default suite test will be used. If using CLI you will be ' - 'led through the questions to configure the experiment.') -@click.option('-p', '--platform', 'platform', type=click.Choice(get_platforms()), - help='If using defaults for input_method, this option is used ' - 'to determine which platform to use for platform specific defaults.') -@click.option('-o', '--override', 'override', default=None, - help='After generating the config file, parameters inside can be overridden ' - 'using values from the override config file.') -@click.option('-t', '--test', 'test', default=None, type=click.Choice(get_suite_tests()), - help='Specify a particular suite test to use for the configuration. This can still ' - 'be overridden.') -def create_experiment(suite, input_method, platform, override, test): - """ - Create a new experiment configuration and directory + type=click.Choice(['defaults', 'cli']), help=input_method_help) +@click.option('-p', '--platform', 'platform', default='nccs_discover', + type=click.Choice(get_platforms()), help=platform_help) +@click.option('-o', '--override', 'override', default=None, help=override_help) +@click.option('-a', '--advanced', 'advanced', default=False, help=advanced_help) +def create(suite, input_method, platform, override, advanced): + """ + Create a new experiment This command creates an experiment directory based on the provided suite name and options. @@ -63,7 +94,39 @@ def create_experiment(suite, input_method, platform, override, test): suite (str): Name of the suite you wish to run. \n """ - experiment_dict_str = prepare_config(suite, input_method, platform, override, test) + # First create the configuration for the experiment. + experiment_dict_str = prepare_config(suite, input_method, platform, override, advanced) + + # Create the experiment directory + create_experiment_directory(experiment_dict_str) + + +# -------------------------------------------------------------------------------------------------- + + +@swell_driver.command() +@click.argument('configuration') +@click.argument('experiment_id') +@click.option('-m', '--input_method', 'input_method', default='defaults', + type=click.Choice(['defaults', 'cli']), help=input_method_help) +@click.option('-p', '--platform', 'platform', default=None, help=platform_help) +@click.option('-a', '--advanced', 'advanced', default=False, help=advanced_help) +def clone(configuration, experiment_id, input_method, platform, advanced): + """ + Clone an existing experiment + + This command creates an experiment directory based on the provided experiment configuration. + + Arguments: \n + configuration (str): Path to a YAML containing the experiment configuration you wish to + clone from. \n + + """ + # Create experiment configuration by cloning from existing experiment + experiment_dict_str = clone_config(configuration, experiment_id, input_method, platform, + advanced) + + # Create the experiment directory create_experiment_directory(experiment_dict_str) @@ -72,13 +135,9 @@ def create_experiment(suite, input_method, platform, override, test): @swell_driver.command() @click.argument('suite_path') -@click.option('-b', '--no-detach', 'no_detach', is_flag=True, default=False, - help='Tells the workflow manager not to detach. That is to say run the entire ' - 'run the entire workflow in the foreground and pass back a return code.') -@click.option('-l', '--log_path', 'log_path', default=None, - help='Directory to receive workflow manager logging output (instead of ' - '$HOME/cylc-run/)') -def launch_experiment(suite_path, no_detach, log_path): +@click.option('-b', '--no-detach', 'no_detach', is_flag=True, default=False, help=no_detach_help) +@click.option('-l', '--log_path', 'log_path', default=None, help=log_path_help) +def launch(suite_path, no_detach, log_path): """ Launch an experiment with the cylc workflow manager @@ -88,7 +147,7 @@ def launch_experiment(suite_path, no_detach, log_path): suite_path (str): Path to where the flow.cylc and associated suite files are located. \n """ - launch(suite_path, no_detach, log_path) + launch_experiment(suite_path, no_detach, log_path) # -------------------------------------------------------------------------------------------------- @@ -97,8 +156,8 @@ def launch_experiment(suite_path, no_detach, log_path): @swell_driver.command() @click.argument('task', type=click.Choice(get_tasks())) @click.argument('config') -@click.option('-d', '--datetime', 'datetime', default=None) -@click.option('-m', '--model', 'model', default=None) +@click.option('-d', '--datetime', 'datetime', default=None, help=datetime_help) +@click.option('-m', '--model', 'model', default=None, help=model_help) def task(task, config, datetime, model): """ Run a workflow task diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index 3ace88a5..9abc856a 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -22,9 +22,15 @@ def get_suites(): # Path to platforms suites_directory = os.path.join(get_swell_path(), 'suites') + # List of suites + suites = [dir for dir in os.listdir(suites_directory) + if os.path.isdir(os.path.join(suites_directory, dir))] + + # Sort list alphabetically + suites = sorted(suites) + # List all directories in platform_directory - return [dir for dir in os.listdir(suites_directory) - if os.path.isdir(os.path.join(suites_directory, dir))] + return suites # -------------------------------------------------------------------------------------------------- @@ -43,6 +49,9 @@ def get_suite_tests(): for suite_test_file in suite_test_files: suite_tests.append(os.path.basename(suite_test_file)[0:-5]) + # Sort list alphabetically + suite_tests = sorted(suite_tests) + # Return list of valid task choices return suite_tests From c1dd2a8373390a438d325786ebefaf4a567261bd Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 21:22:46 -0400 Subject: [PATCH 08/20] remove old references to swell_task --- setup.py | 14 +------------- src/swell/deployment/create_experiment.py | 2 +- src/swell/deployment/prep_config_base.py | 4 ++-- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index 487d9756..b9ad254c 100644 --- a/setup.py +++ b/setup.py @@ -50,19 +50,7 @@ include_package_data=True, entry_points={ 'console_scripts': [ - 'swell = swell.swell:main', - 'swell_task = swell.tasks.base.task_base:main', - 'swell_create_experiment = swell.deployment.bin.swell_create_experiment:main', - 'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main', - 'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main', - 'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main', - # Utilities - 'swell_util_check_jedi_interface_templates = \ - swell.utilities.bin.check_jedi_interface_templates:main', - 'swell_util_task_question_dicts = swell.utilities.bin.task_question_dicts:tq_dicts', - 'swell_util_task_question_dicts_defaults = \ - swell.utilities.bin.task_question_dicts_defaults:tq_dicts_defaults', - 'swell_tests_code = swell.test.code_tests.code_tests:main', + 'swell = swell.swell:main' ], }, ) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 045e7e33..107dc20c 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -162,7 +162,7 @@ def template_modules_file(logger, experiment_dict, exp_suite_path): # Swell bin path # -------------- - swell_bin_path = shutil.which("swell_task") + swell_bin_path = shutil.which("swell") swell_bin_path = os.path.split(swell_bin_path)[0] # Swell lib path diff --git a/src/swell/deployment/prep_config_base.py b/src/swell/deployment/prep_config_base.py index 9e73f3c2..766677ac 100644 --- a/src/swell/deployment/prep_config_base.py +++ b/src/swell/deployment/prep_config_base.py @@ -265,8 +265,8 @@ def open_flow(self): base_task_list = [] model_task_list = [] for line in task_s_lines: - if 'script = "swell_task' in line: - task_name = line.split('"swell_task')[1].split(' ')[1] + if 'script = "swell task' in line: + task_name = line.split('"swell task')[1].split(' ')[1] if '-m' in line: if task_name in model_task_list: continue From c6f6e4c0ba2aba5d1b402d27baf136b1d75832a4 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:01:46 -0400 Subject: [PATCH 09/20] utilities and tests working --- src/swell/deployment/launch_experiment.py | 1 - src/swell/deployment/prep_config.py | 2 +- src/swell/swell.py | 6 +++--- .../question_dictionary_comparison_test.py | 4 ++-- src/swell/test/test_driver.py | 2 ++ .../utilities/scripts/task_question_dicts.py | 2 +- .../scripts/task_question_dicts_defaults.py | 2 +- src/swell/utilities/scripts/utility_driver.py | 19 +++++++++++++++---- 8 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/swell/deployment/launch_experiment.py b/src/swell/deployment/launch_experiment.py index 644b24de..eeebdb81 100644 --- a/src/swell/deployment/launch_experiment.py +++ b/src/swell/deployment/launch_experiment.py @@ -13,7 +13,6 @@ # local imports from swell.utilities.logger import Logger from swell.utilities.shell_commands import run_subprocess -from swell.utilities.welcome_message import write_welcome_message # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_config.py b/src/swell/deployment/prep_config.py index a5193da4..b16fd575 100644 --- a/src/swell/deployment/prep_config.py +++ b/src/swell/deployment/prep_config.py @@ -16,7 +16,7 @@ from swell.utilities.logger import Logger from swell.swell_path import get_swell_path -from swell.utilities.dictionary import dict_get, add_comments_to_dictionary +from swell.utilities.dictionary import add_comments_to_dictionary # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 813723cc..ebb69acc 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -18,6 +18,7 @@ from swell.test.test_driver import test_wrapper, valid_tests from swell.utilities.suite_utils import get_suites from swell.utilities.welcome_message import write_welcome_message +from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper # -------------------------------------------------------------------------------------------------- @@ -176,7 +177,7 @@ def task(task, config, datetime, model): @swell_driver.command() -@click.argument('utility') +@click.argument('utility', type=click.Choice(get_utilities())) def utility(utility): """ Run a utility script @@ -187,7 +188,7 @@ def utility(utility): utility (str): Name of the utility operation to perform.\n """ - print(utility) + utility_wrapper(utility) # -------------------------------------------------------------------------------------------------- @@ -205,7 +206,6 @@ def test(test): test (str): Name of the test to execute. """ - print(test) test_wrapper(test) diff --git a/src/swell/test/code_tests/question_dictionary_comparison_test.py b/src/swell/test/code_tests/question_dictionary_comparison_test.py index 066c99a6..fcec99b9 100644 --- a/src/swell/test/code_tests/question_dictionary_comparison_test.py +++ b/src/swell/test/code_tests/question_dictionary_comparison_test.py @@ -10,8 +10,8 @@ import unittest -from swell.utilities.bin.task_question_dicts import tq_dicts -from swell.utilities.bin.task_question_dicts_defaults import tq_dicts_defaults +from swell.utilities.scripts.task_question_dicts import main as tq_dicts +from swell.utilities.scripts.task_question_dicts_defaults import main as tq_dicts_defaults # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/test/test_driver.py b/src/swell/test/test_driver.py index a0f195be..fa8122c5 100644 --- a/src/swell/test/test_driver.py +++ b/src/swell/test/test_driver.py @@ -27,3 +27,5 @@ def test_wrapper(test): # Run the test test_method() + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/scripts/task_question_dicts.py b/src/swell/utilities/scripts/task_question_dicts.py index 7691a31c..c991cf1d 100644 --- a/src/swell/utilities/scripts/task_question_dicts.py +++ b/src/swell/utilities/scripts/task_question_dicts.py @@ -23,7 +23,7 @@ # -------------------------------------------------------------------------------------------------- -def tq_dicts(): +def main(): # Create a logger logger = Logger('ListOfTaskQuestions') diff --git a/src/swell/utilities/scripts/task_question_dicts_defaults.py b/src/swell/utilities/scripts/task_question_dicts_defaults.py index 6c4ee645..24349d72 100644 --- a/src/swell/utilities/scripts/task_question_dicts_defaults.py +++ b/src/swell/utilities/scripts/task_question_dicts_defaults.py @@ -124,7 +124,7 @@ def create_platform_tq_dicts(logger, platform_name, tq_dicts, platform_tq_dicts_ # -------------------------------------------------------------------------------------------------- -def tq_dicts_defaults(): +def main(): # Create a logger logger = Logger('ListOfTaskQuestions') diff --git a/src/swell/utilities/scripts/utility_driver.py b/src/swell/utilities/scripts/utility_driver.py index 96f56bb1..7f58961c 100644 --- a/src/swell/utilities/scripts/utility_driver.py +++ b/src/swell/utilities/scripts/utility_driver.py @@ -13,12 +13,12 @@ import importlib from swell.swell_path import get_swell_path -from swell.utilities.case_switching import snake_case_to_camel_case -from swell.test.code_tests.code_tests import code_tests +from swell.utilities.case_switching import snake_case_to_camel_case, camel_case_to_snake_case # -------------------------------------------------------------------------------------------------- + def get_utilities(): # Path to util scripts @@ -34,18 +34,29 @@ def get_utilities(): if '__' not in base_name: util_scripts.append(snake_case_to_camel_case(base_name[0:-3])) + # Remove UtilityDriver from util_scripts + util_scripts.remove('UtilityDriver') + # Return list of valid task choices return util_scripts + # -------------------------------------------------------------------------------------------------- + def utility_wrapper(utility): + # Convert utility to snake case + utility_snake = camel_case_to_snake_case(utility) + # Test script - test_script_file = 'swell.utility.scripts.'+utility+'.'+utility + test_script_file = 'swell.utilities.scripts.'+utility_snake # Import the correct method - utility_method = getattr(importlib.import_module(test_script_file), utility) + utility_method = getattr(importlib.import_module(test_script_file), 'main') # Run the test utility_method() + + +# -------------------------------------------------------------------------------------------------- From c5674b9a3a0443adc88d3cd195477b22c0a6d9c7 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:05:49 -0400 Subject: [PATCH 10/20] pycodestyle --- src/swell/test/test_driver.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/swell/test/test_driver.py b/src/swell/test/test_driver.py index fa8122c5..414e89b5 100644 --- a/src/swell/test/test_driver.py +++ b/src/swell/test/test_driver.py @@ -4,19 +4,17 @@ # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - # -------------------------------------------------------------------------------------------------- - import importlib - # -------------------------------------------------------------------------------------------------- valid_tests = ['code_tests'] # -------------------------------------------------------------------------------------------------- + def test_wrapper(test): # Test script @@ -28,4 +26,5 @@ def test_wrapper(test): # Run the test test_method() + # -------------------------------------------------------------------------------------------------- From 79d8a8447fe55df51d599b134abe42e559d26aed Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:22:36 -0400 Subject: [PATCH 11/20] minor rearranging --- src/swell/deployment/create_experiment.py | 100 ++++++++++++- .../platforms.py} | 0 src/swell/deployment/prep_config.py | 138 ------------------ src/swell/swell.py | 6 +- 4 files changed, 102 insertions(+), 142 deletions(-) rename src/swell/deployment/{prep_config_utils.py => platforms/platforms.py} (100%) delete mode 100644 src/swell/deployment/prep_config.py diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 107dc20c..68868460 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -9,12 +9,14 @@ import copy +import datetime +import importlib import os import shutil import yaml from swell.swell_path import get_swell_path -from swell.utilities.dictionary import dict_get +from swell.utilities.dictionary import add_comments_to_dictionary, dict_get from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.logger import Logger @@ -22,6 +24,102 @@ # -------------------------------------------------------------------------------------------------- +def clone_config(configuration, experiment_id, method, platform, advanced): + + # Create a logger + logger = Logger('SwellCloneExperiment') + + # Check that configuration exists and is a YAML file + if not os.path.isfile(configuration): + logger.abort(f'The provided configuration file {configuration} does not exist') + + # Open the target experiment YAML. It will be used as the override + with open(configuration, 'r') as f: + override_dict = yaml.safe_load(f) + + # Check that override_dict has a suite key and get the suite name + if 'suite_to_run' not in override_dict: + logger.abort('The provided configuration file does not have a \'suite_to_run\' key') + suite = override_dict['suite_to_run'] + + # The user may want to run on a different platform (if so adjust the override) + if platform is not None: + override_dict['platform'] = platform + + # Set the experiment_id in the override dictionary + override_dict['experiment_id'] = experiment_id + + # First create the configuration for the experiment. + return prepare_config(suite, method, override_dict['platform'], override_dict, advanced) + + +# -------------------------------------------------------------------------------------------------- + + +def prepare_config(suite, method, platform, override, advanced): + + # Create a logger + # --------------- + logger = Logger('SwellPrepSuiteConfig') + + # Starting point for configuration generation + # ------------------------------------------- + config_file = os.path.join(get_swell_path(), 'suites', 'suite_questions.yaml') + + # Assert valid method + # ------------------- + valid_tasks = ['defaults', 'cli'] + if method not in valid_tasks: + logger.abort(f'In Suites constructor method \'{method}\' not one of the valid ' + + f'tasks {valid_tasks}') + + # Set the object that will be used to populate dictionary options + # --------------------------------------------------------------- + PrepUsing = getattr(importlib.import_module('swell.deployment.prep_config_'+method), + 'PrepConfig'+method.capitalize()) + prep_using = PrepUsing(logger, config_file, suite, platform, override, advanced) + + # Call the config prep step + # ------------------------- + prep_using.execute() + + # Copy the experiment dictionary + # ------------------------------ + experiment_dict = prep_using.experiment_dict + comment_dict = prep_using.comment_dict + + # Add the datetime to the dictionary + # ---------------------------------- + experiment_dict['datetime_created'] = datetime.datetime.today().strftime("%Y%m%d_%H%M%SZ") + comment_dict['datetime_created'] = 'Datetime this file was created (auto added)' + + # Add the model components to the dictionary + # ------------------------------------------ + if 'models' in experiment_dict: + experiment_dict['model_components'] = list(experiment_dict['models'].keys()) + comment_dict['model_components'] = 'List of models in this experiment' + + # Expand all environment vars in the dictionary + # --------------------------------------------- + experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) + experiment_dict_string = os.path.expandvars(experiment_dict_string) + experiment_dict = yaml.safe_load(experiment_dict_string) + + # Add comments to dictionary + # -------------------------- + experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) + + experiment_dict_string_comments = add_comments_to_dictionary(experiment_dict_string, + comment_dict) + + # Return path to dictionary file + # ------------------------------ + return experiment_dict_string_comments + + +# -------------------------------------------------------------------------------------------------- + + def create_experiment_directory(experiment_dict_str): # Create a logger diff --git a/src/swell/deployment/prep_config_utils.py b/src/swell/deployment/platforms/platforms.py similarity index 100% rename from src/swell/deployment/prep_config_utils.py rename to src/swell/deployment/platforms/platforms.py diff --git a/src/swell/deployment/prep_config.py b/src/swell/deployment/prep_config.py deleted file mode 100644 index b16fd575..00000000 --- a/src/swell/deployment/prep_config.py +++ /dev/null @@ -1,138 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import copy -import datetime -import os -import importlib -import yaml - -from swell.utilities.logger import Logger -from swell.swell_path import get_swell_path -from swell.utilities.dictionary import add_comments_to_dictionary - - -# -------------------------------------------------------------------------------------------------- - - -def update_model_components(logger, experiment_dict, comment_dict): - - if 'models' in experiment_dict: - model_components_wanted = copy.copy(experiment_dict['model_components']) - model_components_actual = list(experiment_dict['models'].keys()) - - # If models element of experiment dictionary contains anything not in - # model_components_actual then remove it from model - for model in model_components_actual: - if model not in model_components_wanted: - del (experiment_dict['models'][model]) - # Loop over all elements of the comment dictionay and remove any redundant keys - for key in list(comment_dict.keys()): - if 'models.'+model in key: - del (comment_dict[key]) - - -# -------------------------------------------------------------------------------------------------- - - -def clone_config(configuration, experiment_id, method, platform, advanced): - - # Create a logger - logger = Logger('SwellCloneExperiment') - - # Check that configuration exists and is a YAML file - if not os.path.isfile(configuration): - logger.abort(f'The provided configuration file {configuration} does not exist') - - # Open the target experiment YAML. It will be used as the override - with open(configuration, 'r') as f: - override_dict = yaml.safe_load(f) - - # Check that override_dict has a suite key and get the suite name - if 'suite_to_run' not in override_dict: - logger.abort('The provided configuration file does not have a \'suite_to_run\' key') - suite = override_dict['suite_to_run'] - - # The user may want to run on a different platform (if so adjust the override) - if platform is not None: - override_dict['platform'] = platform - - # Set the experiment_id in the override dictionary - override_dict['experiment_id'] = experiment_id - - # First create the configuration for the experiment. - return prepare_config(suite, method, override_dict['platform'], override_dict, advanced) - - -# -------------------------------------------------------------------------------------------------- - - -def prepare_config(suite, method, platform, override, advanced): - - # Create a logger - # --------------- - logger = Logger('SwellPrepSuiteConfig') - - # Starting point for configuration generation - # ------------------------------------------- - config_file = os.path.join(get_swell_path(), 'suites', 'suite_questions.yaml') - - # Assert valid method - # ------------------- - valid_tasks = ['defaults', 'cli'] - if method not in valid_tasks: - logger.abort(f'In Suites constructor method \'{method}\' not one of the valid ' + - f'tasks {valid_tasks}') - - # Set the object that will be used to populate dictionary options - # --------------------------------------------------------------- - PrepUsing = getattr(importlib.import_module('swell.deployment.prep_config_'+method), - 'PrepConfig'+method.capitalize()) - prep_using = PrepUsing(logger, config_file, suite, platform, override, advanced) - - # Call the config prep step - # ------------------------- - prep_using.execute() - - # Copy the experiment dictionary - # ------------------------------ - experiment_dict = prep_using.experiment_dict - comment_dict = prep_using.comment_dict - - # Add the datetime to the dictionary - # ---------------------------------- - experiment_dict['datetime_created'] = datetime.datetime.today().strftime("%Y%m%d_%H%M%SZ") - comment_dict['datetime_created'] = 'Datetime this file was created (auto added)' - - # Add the model components to the dictionary - # ------------------------------------------ - if 'models' in experiment_dict: - experiment_dict['model_components'] = list(experiment_dict['models'].keys()) - comment_dict['model_components'] = 'List of models in this experiment' - - # Expand all environment vars in the dictionary - # --------------------------------------------- - experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) - experiment_dict_string = os.path.expandvars(experiment_dict_string) - experiment_dict = yaml.safe_load(experiment_dict_string) - - # Add comments to dictionary - # -------------------------- - experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) - - experiment_dict_string_comments = add_comments_to_dictionary(experiment_dict_string, - comment_dict) - - # Return path to dictionary file - # ------------------------------ - return experiment_dict_string_comments - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index ebb69acc..fd720f9a 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -10,9 +10,9 @@ import click -from swell.deployment.prep_config import clone_config, prepare_config -from swell.deployment.prep_config_utils import get_platforms -from swell.deployment.create_experiment import create_experiment_directory +from swell.deployment.platforms.platforms import get_platforms +from swell.deployment.create_experiment import clone_config, create_experiment_directory +from swell.deployment.create_experiment import prepare_config from swell.deployment.launch_experiment import launch_experiment from swell.tasks.base.task_base import task_wrapper, get_tasks from swell.test.test_driver import test_wrapper, valid_tests From fa211a6c361e719641f959b5a16581dce3c5fc09 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:25:32 -0400 Subject: [PATCH 12/20] minor version bumo --- src/swell/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/__init__.py b/src/swell/__init__.py index 11f86a89..e78fcdb1 100644 --- a/src/swell/__init__.py +++ b/src/swell/__init__.py @@ -9,4 +9,4 @@ repo_directory = os.path.dirname(__file__) # Set the version for swell -__version__ = '1.6.4' +__version__ = '1.7.0' From 9d9a5f80a38c2a991bb5ba9b003cf54d850ba1de Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:26:52 -0400 Subject: [PATCH 13/20] gh testing --- .github/workflows/code_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code_tests.yml b/.github/workflows/code_tests.yml index 8ded49cb..1d95827d 100644 --- a/.github/workflows/code_tests.yml +++ b/.github/workflows/code_tests.yml @@ -51,4 +51,4 @@ jobs: # Run the swell code tests - name: Run swell code tests - run: swell_tests_code + run: swell test code_tests From 609647d09050b5e19742d98690e0132d696b6512 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:30:58 -0400 Subject: [PATCH 14/20] add click to gh requirements --- requirements-github.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-github.txt b/requirements-github.txt index b2f1d9c2..359bc62a 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -1,3 +1,4 @@ +click==8.1.5 pyyaml==6.0 pycodestyle==2.10.0 flake8==6.0.0 From d2ac32023d95b53e8dc0869ae2ba9e717cb61066 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:35:19 -0400 Subject: [PATCH 15/20] more gh requirements --- requirements-github.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requirements-github.txt b/requirements-github.txt index 359bc62a..a9d7919c 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -1,4 +1,9 @@ click==8.1.5 +jinja2==3.1.2 pyyaml==6.0 pycodestyle==2.10.0 +pandas==1.4.0 +isodate==0.5.4 +f90nml==1.4.3 +questionary==1.10.0 flake8==6.0.0 From 4e3108869acbef93b5ef6d91ed3b0a6c802e0706 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:40:27 -0400 Subject: [PATCH 16/20] more gh requirements --- requirements-github.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-github.txt b/requirements-github.txt index a9d7919c..23f9a97f 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -2,8 +2,8 @@ click==8.1.5 jinja2==3.1.2 pyyaml==6.0 pycodestyle==2.10.0 -pandas==1.4.0 -isodate==0.5.4 -f90nml==1.4.3 +#pandas==1.4.0 +#isodate==0.6.1 +#f90nml==1.4.3 questionary==1.10.0 flake8==6.0.0 From 45543caa576ab087aa6117a9800c4893369f832a Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:41:55 -0400 Subject: [PATCH 17/20] more gh requirements --- requirements-github.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-github.txt b/requirements-github.txt index 23f9a97f..fd72e4b2 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -3,7 +3,7 @@ jinja2==3.1.2 pyyaml==6.0 pycodestyle==2.10.0 #pandas==1.4.0 -#isodate==0.6.1 +isodate==0.6.1 #f90nml==1.4.3 questionary==1.10.0 flake8==6.0.0 From 3f9aca83218192558bda99c57dfd99b502b2c3f2 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:46:28 -0400 Subject: [PATCH 18/20] more gh requirements --- requirements-github.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-github.txt b/requirements-github.txt index fd72e4b2..d3d935ff 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -4,6 +4,6 @@ pyyaml==6.0 pycodestyle==2.10.0 #pandas==1.4.0 isodate==0.6.1 -#f90nml==1.4.3 +f90nml==1.4.3 questionary==1.10.0 flake8==6.0.0 From 24512720e480be46596e7bbdcefafc5975a86d16 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 4 Oct 2023 22:49:21 -0400 Subject: [PATCH 19/20] more gh requirements --- requirements-github.txt | 1 + requirements-standalone.txt | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-github.txt b/requirements-github.txt index d3d935ff..2f0dbd3a 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -7,3 +7,4 @@ isodate==0.6.1 f90nml==1.4.3 questionary==1.10.0 flake8==6.0.0 +netCDF4==1.6.4 diff --git a/requirements-standalone.txt b/requirements-standalone.txt index bcaaa59a..361b9a3d 100644 --- a/requirements-standalone.txt +++ b/requirements-standalone.txt @@ -8,5 +8,3 @@ f90nml>=1.4.3 questionary>=1.10.0 flake8>=6.0.0 netCDF4 -xarray -matplotlib From 4c011ca7fa5c4b6d98b1ed6a425d62cd78346dd5 Mon Sep 17 00:00:00 2001 From: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> Date: Fri, 6 Oct 2023 13:25:03 -0400 Subject: [PATCH 20/20] Update src/swell/deployment/platforms/generic/suite_questions.yaml Co-authored-by: Colin Egerer --- src/swell/deployment/platforms/generic/suite_questions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/deployment/platforms/generic/suite_questions.yaml b/src/swell/deployment/platforms/generic/suite_questions.yaml index 889ba481..33892d3a 100644 --- a/src/swell/deployment/platforms/generic/suite_questions.yaml +++ b/src/swell/deployment/platforms/generic/suite_questions.yaml @@ -5,4 +5,4 @@ experiment_root: default_value: $HOME/SwellExperiments r2d2_local_path: - default_value: /home/${USER}/R2D2DataStore/Local + default_value: $HOME/R2D2DataStore/Local