diff --git a/.github/workflows/code_tests.yml b/.github/workflows/code_tests.yml index 8ded49cb..1d95827d 100644 --- a/.github/workflows/code_tests.yml +++ b/.github/workflows/code_tests.yml @@ -51,4 +51,4 @@ jobs: # Run the swell code tests - name: Run swell code tests - run: swell_tests_code + run: swell test code_tests diff --git a/requirements-github.txt b/requirements-github.txt index b2f1d9c2..2f0dbd3a 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -1,3 +1,10 @@ +click==8.1.5 +jinja2==3.1.2 pyyaml==6.0 pycodestyle==2.10.0 +#pandas==1.4.0 +isodate==0.6.1 +f90nml==1.4.3 +questionary==1.10.0 flake8==6.0.0 +netCDF4==1.6.4 diff --git a/requirements-standalone.txt b/requirements-standalone.txt index bcaaa59a..361b9a3d 100644 --- a/requirements-standalone.txt +++ b/requirements-standalone.txt @@ -8,5 +8,3 @@ f90nml>=1.4.3 questionary>=1.10.0 flake8>=6.0.0 netCDF4 -xarray -matplotlib diff --git a/setup.py b/setup.py index df7173aa..b9ad254c 100644 --- a/setup.py +++ b/setup.py @@ -50,18 +50,7 @@ include_package_data=True, entry_points={ 'console_scripts': [ - 'swell_task = swell.tasks.base.task_base:main', - 'swell_create_experiment = swell.deployment.bin.swell_create_experiment:main', - 'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main', - 'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main', - 'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main', - # Utilities - 'swell_util_check_jedi_interface_templates = \ - swell.utilities.bin.check_jedi_interface_templates:main', - 'swell_util_task_question_dicts = swell.utilities.bin.task_question_dicts:tq_dicts', - 'swell_util_task_question_dicts_defaults = \ - swell.utilities.bin.task_question_dicts_defaults:tq_dicts_defaults', - 'swell_tests_code = swell.test.code_tests.code_tests:main', + 'swell = swell.swell:main' ], }, ) diff --git a/src/swell/__init__.py b/src/swell/__init__.py index 11f86a89..e78fcdb1 100644 --- a/src/swell/__init__.py +++ b/src/swell/__init__.py @@ -9,4 +9,4 @@ repo_directory = os.path.dirname(__file__) # Set the version for swell -__version__ = '1.6.4' +__version__ = '1.7.0' diff --git a/src/swell/deployment/bin/swell_create_experiment.py b/src/swell/deployment/bin/swell_create_experiment.py deleted file mode 100644 index 4a1ff6ba..00000000 --- a/src/swell/deployment/bin/swell_create_experiment.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python - -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import click -import os -import shutil -import yaml - -from swell.deployment.prep_exp_dirs import copy_eva_files, copy_platform_files, \ - template_modules_file, create_modules_csh -from swell.deployment.prep_suite import prepare_cylc_suite_jinja2 -from swell.swell_path import get_swell_path -from swell.utilities.dictionary import dict_get -from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.logger import Logger -from swell.utilities.welcome_message import write_welcome_message - - -# -------------------------------------------------------------------------------------------------- - - -@click.command() -@click.argument('config_file') -def main(config_file): - - # Welcome message - # --------------- - write_welcome_message('Create Experiment') - - # Create a logger - # --------------- - logger = Logger('SwellCreateExperiment') - - # Load experiment file - # -------------------- - with open(config_file, 'r') as ymlfile: - experiment_dict = yaml.safe_load(ymlfile) - - # Extract from the config - # ----------------------- - experiment_id = dict_get(logger, experiment_dict, 'experiment_id') - experiment_root = dict_get(logger, experiment_dict, 'experiment_root') - platform = dict_get(logger, experiment_dict, 'platform', None) - suite_to_run = dict_get(logger, experiment_dict, 'suite_to_run') - - # Make the suite directory - # ------------------------ - exp_path = os.path.join(experiment_root, experiment_id) - exp_suite_path = os.path.join(exp_path, experiment_id+'-suite') - os.makedirs(exp_suite_path, 0o755, exist_ok=True) - - # Copy experiment file to suite dir - # --------------------------------- - shutil.copyfile(config_file, os.path.join(exp_suite_path, 'experiment.yaml')) - - # Copy suite and platform files to experiment suite directory - # ----------------------------------------------------------- - swell_suite_path = os.path.join(get_swell_path(), 'suites', suite_to_run) - copy_platform_files(logger, exp_suite_path, platform) - - if os.path.exists(os.path.join(swell_suite_path, 'eva')): - copy_eva_files(logger, swell_suite_path, exp_suite_path) - - # Create R2D2 database file - # ------------------------- - r2d2_local_path = dict_get(logger, experiment_dict, 'r2d2_local_path', None) - if r2d2_local_path is not None: - r2d2_conf_path = os.path.join(exp_suite_path, 'r2d2_config.yaml') - - # Write R2D2_CONFIG to modules - with open(os.path.join(exp_suite_path, 'modules'), 'a') as module_file: - module_file.write(f'export R2D2_CONFIG={r2d2_conf_path}') - - # Open the r2d2 file to dictionary - with open(r2d2_conf_path, 'r') as r2d2_file_open: - r2d2_file_str = r2d2_file_open.read() - r2d2_file_str = template_string_jinja2(logger, r2d2_file_str, experiment_dict) - r2d2_file_str = os.path.expandvars(r2d2_file_str) - - with open(r2d2_conf_path, 'w') as r2d2_file_open: - r2d2_file_open.write(r2d2_file_str) - - # Set the swell paths in the modules file and create csh versions - # --------------------------------------------------------------- - template_modules_file(logger, experiment_dict, exp_suite_path) - create_modules_csh(logger, exp_suite_path) - - # Set the jinja2 file for cylc - # ---------------------------- - prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experiment_dict) - - # Copy config directory to experiment - # ----------------------------------- - src = os.path.join(get_swell_path(), 'configuration') - dst = os.path.join(exp_path, 'configuration') - if os.path.exists(dst) and os.path.isdir(dst): - shutil.rmtree(dst) - shutil.copytree(src, dst, ignore=shutil.ignore_patterns('*.py*', '*__*')) - - # Write out launch command for convenience - # ---------------------------------------- - logger.info(' ') - logger.info(' Experiment successfully installed. To launch experiment use: ') - logger.info(' swell_launch_experiment --suite_path ' + exp_suite_path, False) - logger.info(' ') - - -# -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/bin/swell_prepare_config.py b/src/swell/deployment/bin/swell_prepare_config.py deleted file mode 100644 index 426b247e..00000000 --- a/src/swell/deployment/bin/swell_prepare_config.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import click - -from swell.deployment.prep_config import prepare_config -from swell.utilities.welcome_message import write_welcome_message - - -# -------------------------------------------------------------------------------------------------- - - -@click.command() -@click.option('-m', '--input_method', 'input_method', default='defaults', help='Method by which ' + - 'to create the YAML configuration file. Valid choices: \'defaults\', \'cli\'.') -@click.option('-p', '--platform', 'platform', default='nccs_discover', help='If using defaults ' + - 'for input_method this option is used to determine which platform to use for ' + - 'platform specific defaults.') -@click.option('-o', '--override', 'override', default=None, help='After generating the config ' + - 'file parameters inside can be overridden using value from the override config file.') -@click.option('-a', '--advanced', 'advanced', default=False, help='Show configuration questions ' + - 'which are otherwise not shown to the user.') -@click.argument('suite') -def main(input_method, suite, platform, override, advanced): - """ - SUITE argument determines which set of tasks are going to be run. - """ - - # Welcome message - # --------------- - write_welcome_message('Prepare Config') - - # Create suites object - # -------------------- - prepare_config(input_method, suite, platform, override, advanced) - - -# -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py new file mode 100644 index 00000000..a384813b --- /dev/null +++ b/src/swell/deployment/create_experiment.py @@ -0,0 +1,491 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import copy +import datetime +import importlib +import os +import shutil +import yaml + +from swell.swell_path import get_swell_path +from swell.utilities.dictionary import add_comments_to_dictionary, dict_get +from swell.utilities.jinja2 import template_string_jinja2 +from swell.utilities.logger import Logger + + +# -------------------------------------------------------------------------------------------------- + + +def clone_config(configuration, experiment_id, method, platform, advanced): + + # Create a logger + logger = Logger('SwellCloneExperiment') + + # Check that configuration exists and is a YAML file + if not os.path.isfile(configuration): + logger.abort(f'The provided configuration file {configuration} does not exist') + + # Open the target experiment YAML. It will be used as the override + with open(configuration, 'r') as f: + override_dict = yaml.safe_load(f) + + # Check that override_dict has a suite key and get the suite name + if 'suite_to_run' not in override_dict: + logger.abort('The provided configuration file does not have a \'suite_to_run\' key') + suite = override_dict['suite_to_run'] + + # The user may want to run on a different platform (if so adjust the override) + if platform is not None: + override_dict['platform'] = platform + + # Set the experiment_id in the override dictionary + override_dict['experiment_id'] = experiment_id + + # First create the configuration for the experiment. + return prepare_config(suite, method, override_dict['platform'], override_dict, advanced) + + +# -------------------------------------------------------------------------------------------------- + + +def prepare_config(suite, method, platform, override, advanced): + + # Create a logger + # --------------- + logger = Logger('SwellPrepSuiteConfig') + + # Starting point for configuration generation + # ------------------------------------------- + config_file = os.path.join(get_swell_path(), 'suites', 'suite_questions.yaml') + + # Assert valid method + # ------------------- + valid_tasks = ['defaults', 'cli'] + if method not in valid_tasks: + logger.abort(f'In Suites constructor method \'{method}\' not one of the valid ' + + f'tasks {valid_tasks}') + + # Set the object that will be used to populate dictionary options + # --------------------------------------------------------------- + PrepUsing = getattr(importlib.import_module('swell.deployment.prep_config_'+method), + 'PrepConfig'+method.capitalize()) + prep_using = PrepUsing(logger, config_file, suite, platform, override, advanced) + + # Call the config prep step + # ------------------------- + prep_using.execute() + + # Copy the experiment dictionary + # ------------------------------ + experiment_dict = prep_using.experiment_dict + comment_dict = prep_using.comment_dict + + # Add the datetime to the dictionary + # ---------------------------------- + experiment_dict['datetime_created'] = datetime.datetime.today().strftime("%Y%m%d_%H%M%SZ") + comment_dict['datetime_created'] = 'Datetime this file was created (auto added)' + + # Add the model components to the dictionary + # ------------------------------------------ + if 'models' in experiment_dict: + experiment_dict['model_components'] = list(experiment_dict['models'].keys()) + comment_dict['model_components'] = 'List of models in this experiment' + + # Expand all environment vars in the dictionary + # --------------------------------------------- + experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) + experiment_dict_string = os.path.expandvars(experiment_dict_string) + experiment_dict = yaml.safe_load(experiment_dict_string) + + # Add comments to dictionary + # -------------------------- + experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) + + experiment_dict_string_comments = add_comments_to_dictionary(experiment_dict_string, + comment_dict) + + # Return path to dictionary file + # ------------------------------ + return experiment_dict_string_comments + + +# -------------------------------------------------------------------------------------------------- + + +def create_experiment_directory(experiment_dict_str): + + # Create a logger + # --------------- + logger = Logger('SwellCreateExperiment') + + # Load the string using yaml + # -------------------------- + experiment_dict = yaml.safe_load(experiment_dict_str) + + # Extract from the config + # ----------------------- + experiment_id = dict_get(logger, experiment_dict, 'experiment_id') + experiment_root = dict_get(logger, experiment_dict, 'experiment_root') + platform = dict_get(logger, experiment_dict, 'platform', None) + suite_to_run = dict_get(logger, experiment_dict, 'suite_to_run') + + # Write out some info + # ------------------- + logger.info(f'Creating experiment: \'{experiment_id}\' in \'{experiment_root}\'') + + # Make the suite directory + # ------------------------ + exp_path = os.path.join(experiment_root, experiment_id) + exp_suite_path = os.path.join(exp_path, experiment_id+'-suite') + + os.makedirs(exp_suite_path, 0o755, exist_ok=True) + + # Write dictionary (with comments) to YAML file + # --------------------------------------------- + with open(os.path.join(exp_suite_path, 'experiment.yaml'), 'w') as file: + file.write(experiment_dict_str) + + # Copy suite and platform files to experiment suite directory + # ----------------------------------------------------------- + swell_suite_path = os.path.join(get_swell_path(), 'suites', suite_to_run) + copy_platform_files(logger, exp_suite_path, platform) + + if os.path.exists(os.path.join(swell_suite_path, 'eva')): + copy_eva_files(logger, swell_suite_path, exp_suite_path) + + # Create R2D2 database file + # ------------------------- + r2d2_local_path = dict_get(logger, experiment_dict, 'r2d2_local_path', None) + if r2d2_local_path is not None: + r2d2_conf_path = os.path.join(exp_suite_path, 'r2d2_config.yaml') + + # Write R2D2_CONFIG to modules + with open(os.path.join(exp_suite_path, 'modules'), 'a') as module_file: + module_file.write(f'export R2D2_CONFIG={r2d2_conf_path}') + + # Open the r2d2 file to dictionary + with open(r2d2_conf_path, 'r') as r2d2_file_open: + r2d2_file_str = r2d2_file_open.read() + r2d2_file_str = template_string_jinja2(logger, r2d2_file_str, experiment_dict) + r2d2_file_str = os.path.expandvars(r2d2_file_str) + + with open(r2d2_conf_path, 'w') as r2d2_file_open: + r2d2_file_open.write(r2d2_file_str) + + # Set the swell paths in the modules file and create csh versions + # --------------------------------------------------------------- + template_modules_file(logger, experiment_dict, exp_suite_path) + create_modules_csh(logger, exp_suite_path) + + # Set the jinja2 file for cylc + # ---------------------------- + prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experiment_dict) + + # Copy config directory to experiment + # ----------------------------------- + src = os.path.join(get_swell_path(), 'configuration') + dst = os.path.join(exp_path, 'configuration') + if os.path.exists(dst) and os.path.isdir(dst): + shutil.rmtree(dst) + shutil.copytree(src, dst, ignore=shutil.ignore_patterns('*.py*', '*__*')) + + # Write out launch command for convenience + # ---------------------------------------- + logger.info(' ') + logger.info('Experiment successfully installed. To launch experiment use: ') + logger.info(' ', False) + logger.info(' swell launch ' + exp_suite_path, False) + logger.info(' ', False) + + +# -------------------------------------------------------------------------------------------------- + + +def copy_eva_files(logger, swell_suite_path, exp_suite_path): + + # Repo eva files + eva_directory = os.path.join(swell_suite_path, 'eva') + + # Destination for eva files + destination_directory = os.path.join(exp_suite_path, 'eva') + + # If destination directory exists, delete it + if os.path.exists(destination_directory): + shutil.rmtree(destination_directory) + + # Copy all the files + shutil.copytree(eva_directory, destination_directory) + + +# -------------------------------------------------------------------------------------------------- + + +def copy_platform_files(logger, exp_suite_path, platform=None): + + # Copy platform related files to the suite directory + # -------------------------------------------------- + if platform is not None: + swell_lib_path = get_swell_path() + platform_path = os.path.join(swell_lib_path, 'deployment', 'platforms', platform) + + for s in ['modules', 'r2d2_config.yaml']: + src_file = os.path.split(s)[1] + src_path_file = os.path.join(platform_path, os.path.split(s)[0], src_file) + dst_path_file = os.path.join(exp_suite_path, '{}'.format(src_file)) + if os.path.exists(src_path_file): + logger.trace('Copying {} to {}'.format(src_path_file, dst_path_file)) + shutil.copy(src_path_file, dst_path_file) + + +# -------------------------------------------------------------------------------------------------- + + +def template_modules_file(logger, experiment_dict, exp_suite_path): + + # Modules file + # ------------ + modules_file = os.path.join(exp_suite_path, 'modules') + + # Only do if the suite needs modules + # ---------------------------------- + if os.path.exists(modules_file): + + # Swell bin path + # -------------- + swell_bin_path = shutil.which("swell") + swell_bin_path = os.path.split(swell_bin_path)[0] + + # Swell lib path + # -------------- + swell_lib_path = get_swell_path() + swell_lib_path = os.path.split(swell_lib_path)[0] + + # Swell suite path + # ---------------- + swell_sui_path = os.path.join(get_swell_path(), 'suites') + + # Dictionary of definitions + # ------------------------- + modules_dict = copy.copy(experiment_dict) + modules_dict['swell_bin_path'] = swell_bin_path + modules_dict['swell_lib_path'] = swell_lib_path + modules_dict['swell_sui_path'] = swell_sui_path + + # Open the file + # ------------- + with open(modules_file, 'r') as modules_file_open: + modules_file_str = modules_file_open.read() + + # Resolve templates + # ----------------- + modules_file_str = template_string_jinja2(logger, modules_file_str, modules_dict) + + # Overwrite the file + # ------------------ + with open(modules_file, 'w') as modules_file_open: + modules_file_open.write(modules_file_str) + + +# -------------------------------------------------------------------------------------------------- + + +def create_modules_csh(logger, exp_suite_path): + + # Modules file + # ------------ + modules_file = os.path.join(exp_suite_path, 'modules') + + # Only do if the suite needs modules + # ---------------------------------- + if os.path.exists(modules_file): + + # Open the file + # ------------- + with open(modules_file, 'r') as modules_file_open: + modules_file_lines = modules_file_open.readlines() + + # Replace some things + # ------------------- + for idx, modules_file_line in enumerate(modules_file_lines): + + # 'bash' to 'csh' + if 'bash' in modules_file_line: + modules_file_lines[idx] = modules_file_lines[idx].replace('bash', 'csh') + + # Export to setenv + if 'export' in modules_file_line: + modules_file_lines[idx] = modules_file_lines[idx].replace('export', 'setenv') + modules_file_lines[idx] = modules_file_lines[idx].replace('=', ' ') + + # Set PYTHONPATH + if 'PYTHONPATH=' in modules_file_line: + modules_file_lines[idx] = modules_file_lines[idx].replace('PYTHONPATH=', + 'setenv PYTHONPATH ') + + # Set path + if 'PATH=' in modules_file_line: + modules_file_lines[idx] = modules_file_lines[idx].replace('PATH=', 'set path = (') + modules_file_lines[idx] = modules_file_lines[idx].replace(':$PATH', ' $path)') + + # Overwrite the file + # ------------------ + with open(modules_file+'-csh', 'w') as modules_file_open: + for modules_file_line in modules_file_lines: + modules_file_open.write(modules_file_line) + + +# -------------------------------------------------------------------------------------------------- + + +def prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experiment_dict): + + # Open suite file from swell + # -------------------------- + with open(os.path.join(swell_suite_path, 'flow.cylc'), 'r') as file: + suite_file = file.read() + + # Copy the experiment dictionary to the rendering dictionary + # ---------------------------------------------------------- + render_dictionary = {} + + # Elements to copy from the experiment dictionary + # ----------------------------------------------- + render_elements = [ + 'start_cycle_point', + 'final_cycle_point', + 'runahead_limit', + 'model_components', + 'platform', + ] + + # Copy elements from experiment dictionary to render dictionary + # ------------------------------------------------------------- + for element in render_elements: + if element in experiment_dict: + render_dictionary[element] = experiment_dict[element] + + # Get unique list of cycle times with model flags to render dictionary + # -------------------------------------------------------------------- + + # Check if 'cycle_times' appears anywhere in the suite_file + if 'cycle_times' in suite_file: + + # Since cycle times are used, the render_dictionary will need to include cycle_times + model_components = dict_get(logger, experiment_dict, 'model_components', []) + + # If there are different model components then process each to gather cycle times + if len(model_components) > 0: + cycle_times = [] + for model_component in model_components: + cycle_times_mc = experiment_dict['models'][model_component]['cycle_times'] + cycle_times = list(set(cycle_times + cycle_times_mc)) + cycle_times.sort() + + cycle_times_dict_list = [] + for cycle_time in cycle_times: + cycle_time_dict = {} + cycle_time_dict['cycle_time'] = cycle_time + for model_component in model_components: + cycle_time_dict[model_component] = False + if cycle_time in experiment_dict['models'][model_component]['cycle_times']: + cycle_time_dict[model_component] = True + cycle_times_dict_list.append(cycle_time_dict) + + render_dictionary['cycle_times'] = cycle_times_dict_list + + # Otherwise check that experiment_dict has cycle_times + elif 'cycle_times' in experiment_dict: + + cycle_times = list(set(experiment_dict['cycle_times'])) + cycle_times.sort() + render_dictionary['cycle_times'] = cycle_times + + else: + + # Otherwise use logger to abort + logger.abort('The suite file required cycle_times but there are no model components ' + + 'to gather them from or they are not provided in the experiment ' + + 'dictionary.') + + # Look for a file called $HOME/.swell/slurm.yaml + # ---------------------------------------------- + yaml_path = os.path.expanduser("~/.swell/swell-slurm.yaml") + slurm_global = {} + if os.path.exists(yaml_path): + with open(yaml_path, "r") as yaml_file: + slurm_global = yaml.safe_load(yaml_file) + + # Set default values for global slurm values + account = 'g0613' + qos = 'allnccs' + partition = None + constraint = 'cas|sky' + + # Extract from slurm global file + if 'qos' in slurm_global: + qos = slurm_global['qos'] + + if 'partition' in slurm_global: + partition = slurm_global['partition'] + + if 'account' in slurm_global: + account = slurm_global['account'] + + if 'constraint' in slurm_global: + constraint = slurm_global['constraint'] + + # List of tasks using slurm + # ------------------------- + slurm_tasks = [ + 'BuildJedi', + 'BuildGeos', + 'EvaObservations', + 'GenerateBClimatology', + 'RunJediHofxExecutable', + 'RunJediVariationalExecutable', + 'RunJediUfoTestsExecutable', + 'RunGeosExecutable', + ] + + # Fill default values for slurm tasks + # ----------------------------------- + render_dictionary['scheduling'] = {} + for slurm_task in slurm_tasks: + render_dictionary['scheduling'][slurm_task] = {} + render_dictionary['scheduling'][slurm_task]['execution_time_limit'] = 'PT1H' + render_dictionary['scheduling'][slurm_task]['account'] = account + render_dictionary['scheduling'][slurm_task]['qos'] = qos + render_dictionary['scheduling'][slurm_task]['nodes'] = 1 + render_dictionary['scheduling'][slurm_task]['ntasks_per_node'] = 24 + render_dictionary['scheduling'][slurm_task]['constraint'] = constraint + render_dictionary['scheduling'][slurm_task]['partition'] = partition + + # Set some specific values for: + # ----------------------------- + + # run time + render_dictionary['scheduling']['BuildJedi']['execution_time_limit'] = 'PT3H' + render_dictionary['scheduling']['EvaObservations']['execution_time_limit'] = 'PT30M' + + # nodes + render_dictionary['scheduling']['RunJediUfoTestsExecutable']['ntasks_per_node'] = 1 + + # Render the template + # ------------------- + new_suite_file = template_string_jinja2(logger, suite_file, render_dictionary) + + # Write suite file to experiment + # ------------------------------ + with open(os.path.join(exp_suite_path, 'flow.cylc'), 'w') as file: + file.write(new_suite_file) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/bin/swell_launch_experiment.py b/src/swell/deployment/launch_experiment.py similarity index 81% rename from src/swell/deployment/bin/swell_launch_experiment.py rename to src/swell/deployment/launch_experiment.py index 6e75825a..eeebdb81 100644 --- a/src/swell/deployment/bin/swell_launch_experiment.py +++ b/src/swell/deployment/launch_experiment.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -10,13 +8,11 @@ # standard imports -import click import os # local imports from swell.utilities.logger import Logger from swell.utilities.shell_commands import run_subprocess -from swell.utilities.welcome_message import write_welcome_message # -------------------------------------------------------------------------------------------------- @@ -97,20 +93,7 @@ def cylc_run_experiment(self): # NB: Could be a factory based on workflow_manag # -------------------------------------------------------------------------------------------------- -@click.command() -@click.option('-p', '--suite_path', 'suite_path', default=None, - help='Directory containing the suite file needed by the workflow manager') -@click.option('-w', '--workflow_manager', 'workflow_manager', default='cylc', - help='Workflow manager to be used') -@click.option('-b', '--no-detach', 'no_detach', is_flag=True, default=False, - help='Tells workflow manager to block until complete') -@click.option('-l', '--log_path', 'log_path', default=None, - help='Directory to receive workflow manager logging output') -def main(suite_path, workflow_manager, no_detach, log_path): - - # Welcome message - # --------------- - write_welcome_message('Launch Experiment') +def launch_experiment(suite_path, no_detach, log_path): # Get the path to where the suite files are located # ------------------------------------------------- @@ -132,19 +115,10 @@ def main(suite_path, workflow_manager, no_detach, log_path): deploy_workflow.logger.info('Launching workflow defined by files in \'' + suite_path + '\'.', False) deploy_workflow.logger.info('Experiment name: ' + experiment_name) - deploy_workflow.logger.info('Workflow manager: ' + workflow_manager) # Launch the workflow # ------------------- - if workflow_manager == 'cylc': - deploy_workflow.cylc_run_experiment() - - -# -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() + deploy_workflow.cylc_run_experiment() # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/platforms/generic/suite_questions.yaml b/src/swell/deployment/platforms/generic/suite_questions.yaml index 2793cbf0..33892d3a 100644 --- a/src/swell/deployment/platforms/generic/suite_questions.yaml +++ b/src/swell/deployment/platforms/generic/suite_questions.yaml @@ -2,7 +2,7 @@ experiment_id: default_value: swell-{{suite_to_run}} experiment_root: - default_value: /home/${USER}/SwellExperiments + default_value: $HOME/SwellExperiments r2d2_local_path: - default_value: /home/${USER}/R2D2DataStore/Local + default_value: $HOME/R2D2DataStore/Local diff --git a/src/swell/deployment/platforms/platforms.py b/src/swell/deployment/platforms/platforms.py new file mode 100644 index 00000000..20c89f90 --- /dev/null +++ b/src/swell/deployment/platforms/platforms.py @@ -0,0 +1,34 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os + +from swell.swell_path import get_swell_path + + +# -------------------------------------------------------------------------------------------------- + + +def get_platforms(): + + # Path to platforms + platform_directory = os.path.join(get_swell_path(), 'deployment', 'platforms') + + platforms = [dir for dir in os.listdir(platform_directory) + if os.path.isdir(os.path.join(platform_directory, dir))] + + # If anything in platforms contains '__' remove it from platforms list + platforms = [platform for platform in platforms if '__' not in platform] + + # List all directories in platform_directory + return platforms + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_config.py b/src/swell/deployment/prep_config.py deleted file mode 100644 index 4e5a1b4d..00000000 --- a/src/swell/deployment/prep_config.py +++ /dev/null @@ -1,119 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import copy -import datetime -import os -import importlib -import yaml - -from swell.utilities.logger import Logger -from swell.swell_path import get_swell_path -from swell.utilities.dictionary import dict_get, add_comments_to_dictionary - - -# -------------------------------------------------------------------------------------------------- - - -def update_model_components(logger, experiment_dict, comment_dict): - - if 'models' in experiment_dict: - model_components_wanted = copy.copy(experiment_dict['model_components']) - model_components_actual = list(experiment_dict['models'].keys()) - - # If models element of experiment dictionary contains anything not in - # model_components_actual then remove it from model - for model in model_components_actual: - if model not in model_components_wanted: - logger.info(f'Removing model {model} from model_components') - del (experiment_dict['models'][model]) - # Loop over all elements of the comment dictionay and remove any redundant keys - for key in list(comment_dict.keys()): - if 'models.'+model in key: - del (comment_dict[key]) - - -# -------------------------------------------------------------------------------------------------- - - -def prepare_config(method, suite, platform, override, advanced): - - # Create a logger - # --------------- - logger = Logger('SwellPrepSuiteConfig') - - # Starting point for configuration generation - # ------------------------------------------- - config_file = os.path.join(get_swell_path(), 'suites', 'suite_questions.yaml') - - # Assert valid method - # ------------------- - valid_tasks = ['defaults', 'cli'] - if method not in valid_tasks: - logger.abort(f'In Suites constructor method \'{method}\' not one of the valid ' + - f'tasks {valid_tasks}') - - # Set the object that will be used to populate dictionary options - # --------------------------------------------------------------- - PrepUsing = getattr(importlib.import_module('swell.deployment.prep_config_'+method), - 'PrepConfig'+method.capitalize()) - prep_using = PrepUsing(logger, config_file, suite, platform, override, advanced) - - # Call the config prep step - # ------------------------- - prep_using.execute() - - # Copy the experiment dictionary - # ------------------------------ - experiment_dict = prep_using.experiment_dict - comment_dict = prep_using.comment_dict - - # Add the datetime to the dictionary - # ---------------------------------- - experiment_dict['datetime_created'] = datetime.datetime.today().strftime("%Y%m%d_%H%M%SZ") - comment_dict['datetime_created'] = 'Datetime this file was created (auto added)' - - # Add the model components to the dictionary - # ------------------------------------------ - if 'models' in experiment_dict: - experiment_dict['model_components'] = list(experiment_dict['models'].keys()) - comment_dict['model_components'] = 'List of models in this experiment' - - # Expand all environment vars in the dictionary - # --------------------------------------------- - experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) - experiment_dict_string = os.path.expandvars(experiment_dict_string) - experiment_dict = yaml.safe_load(experiment_dict_string) - - # Add comments to dictionary - # -------------------------- - experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) - - experiment_dict_string_comments = add_comments_to_dictionary(experiment_dict_string, - comment_dict) - - # Dictionary file to write - # ------------------------ - cwd = os.getcwd() - experiment_id = dict_get(logger, experiment_dict, 'experiment_id') - exp_dict_file = os.path.join(cwd, f'{experiment_id}.yaml') - - # Write dictionary to YAML file - # ----------------------------- - with open(exp_dict_file, 'w') as file: - file.write(experiment_dict_string_comments) - logger.info(f'Prepared configuration file written to {exp_dict_file}', False) - - # Return path to dictionary file - # ------------------------------ - return exp_dict_file - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_config_base.py b/src/swell/deployment/prep_config_base.py index 7d7a5e42..766677ac 100644 --- a/src/swell/deployment/prep_config_base.py +++ b/src/swell/deployment/prep_config_base.py @@ -16,6 +16,7 @@ from swell.swell_path import get_swell_path + # -------------------------------------------------------------------------------------------------- @@ -105,12 +106,18 @@ def __init__(self, logger, dictionary_file, suite, platform, override, advanced) # Open user selected override dictionary if override is not None: - logger.info(f'Overriding experiment dictionary settings using {override}') - - select_override_file = os.path.join(override) + logger.info(f'Overriding experiment dictionary settings using override dictionary') - with open(select_override_file, 'r') as select_override_yml: - self.override_dictionary = yaml.safe_load(select_override_yml) + # If override is a dictionary then use it directly + if isinstance(override, dict): + self.override_dictionary = override + elif isinstance(override, str): + # If override is a string then assume it is a path to a yaml file + with open(override, 'r') as select_override_yml: + self.override_dictionary = yaml.safe_load(select_override_yml) + else: + logger.abort(f'Override must be a dictionary or a path to a yaml file. ' + + f'Instead it is {type(override)}') self.override_models() @@ -122,8 +129,8 @@ def execute(self): # defaults gets nothing if self.prep_using == 'Cli': - print("Please answer the following questions to generate your experiment " + - "configuration YAML file.\n") + self.logger("Please answer the following questions to generate your experiment " + + "configuration YAML file.\n") # Set current dictionary variable which is needed for answer changes self.current_dictionary = {} @@ -258,8 +265,8 @@ def open_flow(self): base_task_list = [] model_task_list = [] for line in task_s_lines: - if 'script = "swell_task' in line: - task_name = line.split('"swell_task')[1].split(' ')[1] + if 'script = "swell task' in line: + task_name = line.split('"swell task')[1].split(' ')[1] if '-m' in line: if task_name in model_task_list: continue @@ -269,10 +276,6 @@ def open_flow(self): continue base_task_list.append(task_name) - # The following lists of tasks added to logger - self.logger.info(f'Base tasks selected include the following: {base_task_list}. \n' + - f'Model tasks selected include the following: {model_task_list}.') - return base_task_list, model_task_list # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_exp_dirs.py b/src/swell/deployment/prep_exp_dirs.py deleted file mode 100644 index d45863c2..00000000 --- a/src/swell/deployment/prep_exp_dirs.py +++ /dev/null @@ -1,155 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import copy -import os -import shutil - -from swell.swell_path import get_swell_path -from swell.utilities.jinja2 import template_string_jinja2 - - -# -------------------------------------------------------------------------------------------------- - - -def copy_eva_files(logger, swell_suite_path, exp_suite_path): - - # Repo eva files - eva_directory = os.path.join(swell_suite_path, 'eva') - - # Destination for eva files - destination_directory = os.path.join(exp_suite_path, 'eva') - - # If destination directory exists, delete it - if os.path.exists(destination_directory): - shutil.rmtree(destination_directory) - - # Copy all the files - shutil.copytree(eva_directory, destination_directory) - - -# -------------------------------------------------------------------------------------------------- - - -def copy_platform_files(logger, exp_suite_path, platform=None): - - # Copy platform related files to the suite directory - # -------------------------------------------------- - if platform is not None: - swell_lib_path = get_swell_path() - platform_path = os.path.join(swell_lib_path, 'deployment', 'platforms', platform) - - for s in ['modules', 'r2d2_config.yaml']: - src_file = os.path.split(s)[1] - src_path_file = os.path.join(platform_path, os.path.split(s)[0], src_file) - dst_path_file = os.path.join(exp_suite_path, '{}'.format(src_file)) - if os.path.exists(src_path_file): - logger.trace('Copying {} to {}'.format(src_path_file, dst_path_file)) - shutil.copy(src_path_file, dst_path_file) - - -# -------------------------------------------------------------------------------------------------- - - -def template_modules_file(logger, experiment_dict, exp_suite_path): - - # Modules file - # ------------ - modules_file = os.path.join(exp_suite_path, 'modules') - - # Only do if the suite needs modules - # ---------------------------------- - if os.path.exists(modules_file): - - # Swell bin path - # -------------- - swell_bin_path = shutil.which("swell_task") - swell_bin_path = os.path.split(swell_bin_path)[0] - - # Swell lib path - # -------------- - swell_lib_path = get_swell_path() - swell_lib_path = os.path.split(swell_lib_path)[0] - - # Swell suite path - # ---------------- - swell_sui_path = os.path.join(get_swell_path(), 'suites') - - # Dictionary of definitions - # ------------------------- - modules_dict = copy.copy(experiment_dict) - modules_dict['swell_bin_path'] = swell_bin_path - modules_dict['swell_lib_path'] = swell_lib_path - modules_dict['swell_sui_path'] = swell_sui_path - - # Open the file - # ------------- - with open(modules_file, 'r') as modules_file_open: - modules_file_str = modules_file_open.read() - - # Resolve templates - # ----------------- - modules_file_str = template_string_jinja2(logger, modules_file_str, modules_dict) - - # Overwrite the file - # ------------------ - with open(modules_file, 'w') as modules_file_open: - modules_file_open.write(modules_file_str) - - -# -------------------------------------------------------------------------------------------------- - - -def create_modules_csh(logger, exp_suite_path): - - # Modules file - # ------------ - modules_file = os.path.join(exp_suite_path, 'modules') - - # Only do if the suite needs modules - # ---------------------------------- - if os.path.exists(modules_file): - - # Open the file - # ------------- - with open(modules_file, 'r') as modules_file_open: - modules_file_lines = modules_file_open.readlines() - - # Replace some things - # ------------------- - for idx, modules_file_line in enumerate(modules_file_lines): - - # 'bash' to 'csh' - if 'bash' in modules_file_line: - modules_file_lines[idx] = modules_file_lines[idx].replace('bash', 'csh') - - # Export to setenv - if 'export' in modules_file_line: - modules_file_lines[idx] = modules_file_lines[idx].replace('export', 'setenv') - modules_file_lines[idx] = modules_file_lines[idx].replace('=', ' ') - - # Set PYTHONPATH - if 'PYTHONPATH=' in modules_file_line: - modules_file_lines[idx] = modules_file_lines[idx].replace('PYTHONPATH=', - 'setenv PYTHONPATH ') - - # Set path - if 'PATH=' in modules_file_line: - modules_file_lines[idx] = modules_file_lines[idx].replace('PATH=', 'set path = (') - modules_file_lines[idx] = modules_file_lines[idx].replace(':$PATH', ' $path)') - - # Overwrite the file - # ------------------ - with open(modules_file+'-csh', 'w') as modules_file_open: - for modules_file_line in modules_file_lines: - modules_file_open.write(modules_file_line) - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_suite.py b/src/swell/deployment/prep_suite.py deleted file mode 100644 index 230b4c48..00000000 --- a/src/swell/deployment/prep_suite.py +++ /dev/null @@ -1,163 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import os -import yaml - -from swell.utilities.dictionary import dict_get -from swell.utilities.jinja2 import template_string_jinja2 - - -# -------------------------------------------------------------------------------------------------- - - -def prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experiment_dict): - - # Open suite file from swell - # -------------------------- - with open(os.path.join(swell_suite_path, 'flow.cylc'), 'r') as file: - suite_file = file.read() - - # Copy the experiment dictionary to the rendering dictionary - # ---------------------------------------------------------- - render_dictionary = {} - - # Elements to copy from the experiment dictionary - # ----------------------------------------------- - render_elements = [ - 'start_cycle_point', - 'final_cycle_point', - 'runahead_limit', - 'model_components', - 'platform', - ] - - # Copy elements from experiment dictionary to render dictionary - # ------------------------------------------------------------- - for element in render_elements: - if element in experiment_dict: - render_dictionary[element] = experiment_dict[element] - - # Get unique list of cycle times with model flags to render dictionary - # -------------------------------------------------------------------- - - # Check if 'cycle_times' appears anywhere in the suite_file - if 'cycle_times' in suite_file: - - # Since cycle times are used, the render_dictionary will need to include cycle_times - model_components = dict_get(logger, experiment_dict, 'model_components', []) - - # If there are different model components then process each to gather cycle times - if len(model_components) > 0: - cycle_times = [] - for model_component in model_components: - cycle_times_mc = experiment_dict['models'][model_component]['cycle_times'] - cycle_times = list(set(cycle_times + cycle_times_mc)) - cycle_times.sort() - - cycle_times_dict_list = [] - for cycle_time in cycle_times: - cycle_time_dict = {} - cycle_time_dict['cycle_time'] = cycle_time - for model_component in model_components: - cycle_time_dict[model_component] = False - if cycle_time in experiment_dict['models'][model_component]['cycle_times']: - cycle_time_dict[model_component] = True - cycle_times_dict_list.append(cycle_time_dict) - - render_dictionary['cycle_times'] = cycle_times_dict_list - - # Otherwise check that experiment_dict has cycle_times - elif 'cycle_times' in experiment_dict: - - cycle_times = list(set(experiment_dict['cycle_times'])) - cycle_times.sort() - render_dictionary['cycle_times'] = cycle_times - - else: - - # Otherwise use logger to abort - logger.abort('The suite file required cycle_times but there are no model components ' + - 'to gather them from or they are not provided in the experiment ' + - 'dictionary.') - - # Look for a file called $HOME/.swell/slurm.yaml - # ---------------------------------------------- - yaml_path = os.path.expanduser("~/.swell/swell-slurm.yaml") - slurm_global = {} - if os.path.exists(yaml_path): - logger.info(f'Found file containing swell slurm global values') - with open(yaml_path, "r") as yaml_file: - slurm_global = yaml.safe_load(yaml_file) - - # Set default values for global slurm values - account = 'g0613' - qos = 'allnccs' - partition = None - constraint = 'cas|sky' - - # Extract from slurm global file - if 'qos' in slurm_global: - qos = slurm_global['qos'] - - if 'partition' in slurm_global: - partition = slurm_global['partition'] - - if 'account' in slurm_global: - account = slurm_global['account'] - - if 'constraint' in slurm_global: - constraint = slurm_global['constraint'] - - # List of tasks using slurm - # ------------------------- - slurm_tasks = [ - 'BuildJedi', - 'BuildGeos', - 'EvaObservations', - 'GenerateBClimatology', - 'RunJediHofxExecutable', - 'RunJediLetkfExecutable', - 'RunJediVariationalExecutable', - 'RunJediUfoTestsExecutable', - 'RunGeosExecutable', - ] - - # Fill default values for slurm tasks - # ----------------------------------- - render_dictionary['scheduling'] = {} - for slurm_task in slurm_tasks: - render_dictionary['scheduling'][slurm_task] = {} - render_dictionary['scheduling'][slurm_task]['execution_time_limit'] = 'PT1H' - render_dictionary['scheduling'][slurm_task]['account'] = account - render_dictionary['scheduling'][slurm_task]['qos'] = qos - render_dictionary['scheduling'][slurm_task]['nodes'] = 1 - render_dictionary['scheduling'][slurm_task]['ntasks_per_node'] = 40 - render_dictionary['scheduling'][slurm_task]['constraint'] = constraint - render_dictionary['scheduling'][slurm_task]['partition'] = partition - - # Set some specific values for: - # ----------------------------- - - # run time - render_dictionary['scheduling']['BuildJedi']['execution_time_limit'] = 'PT3H' - render_dictionary['scheduling']['EvaObservations']['execution_time_limit'] = 'PT30M' - - # Render the template - # ------------------- - new_suite_file = template_string_jinja2(logger, suite_file, render_dictionary) - - # Write suite file to experiment - # ------------------------------ - with open(os.path.join(exp_suite_path, 'flow.cylc'), 'w') as file: - file.write(new_suite_file) - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 46aa92ad..7abb1214 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -101,13 +101,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -123,22 +123,22 @@ {% for model_component in model_components %} [[StageJedi-{{model_component}}]] - script = "swell_task StageJedi $config -m {{model_component}}" + script = "swell task StageJedi $config -m {{model_component}}" [[StageJediCycle-{{model_component}}]] - script = "swell_task StageJedi $config -d $datetime -m {{model_component}}" + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" [[ GetBackground-{{model_component}} ]] - script = "swell_task GetBackground $config -d $datetime -m {{model_component}}" + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" [[GetObservations-{{model_component}}]] - script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" [[GenerateBClimatologyByLinking-{{model_component}}]] - script = "swell_task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" + script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell_task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -153,10 +153,10 @@ {% endif %} [[EvaJediLog-{{model_component}}]] - script = "swell_task EvaJediLog $config -d $datetime -m {{model_component}}" + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" [[EvaObservations-{{model_component}}]] - script = "swell_task EvaObservations $config -d $datetime -m {{model_component}}" + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} [[[directives]]] @@ -171,10 +171,10 @@ {% endif %} [[SaveObsDiags-{{model_component}}]] - script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" [[CleanCycle-{{model_component}}]] - script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index 6aa69432..8b95e9b1 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -133,13 +133,13 @@ # Tasks # ----- [[CloneGeos]] - script = "swell_task CloneGeos $config" + script = "swell task CloneGeos $config" [[BuildGeosByLinking]] - script = "swell_task BuildGeosByLinking $config" + script = "swell task BuildGeosByLinking $config" [[BuildGeos]] - script = "swell_task BuildGeos $config" + script = "swell task BuildGeos $config" platform = {{platform}} execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} [[[directives]]] @@ -153,13 +153,13 @@ {% endif %} [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -174,7 +174,7 @@ {% endif %} [[RunGeosExecutable]] - script = "swell_task RunGeosExecutable $config -d $datetime" + script = "swell task RunGeosExecutable $config -d $datetime" platform = {{platform}} execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -189,39 +189,39 @@ {% endif %} [[PrepGeosRunDir]] - script = "swell_task PrepGeosRunDir $config -d $datetime" + script = "swell task PrepGeosRunDir $config -d $datetime" [[RemoveForecastDir]] - script = "swell_task RemoveForecastDir $config -d $datetime" + script = "swell task RemoveForecastDir $config -d $datetime" [[GetGeosRestart]] - script = "swell_task GetGeosRestart $config -d $datetime" + script = "swell task GetGeosRestart $config -d $datetime" {% for model_component in model_components %} [[LinkGeosOutput-{{model_component}}]] - script = "swell_task LinkGeosOutput $config -d $datetime -m {{model_component}}" + script = "swell task LinkGeosOutput $config -d $datetime -m {{model_component}}" [[MoveDaRestart-{{model_component}}]] - script = "swell_task MoveDaRestart $config -d $datetime -m {{model_component}}" + script = "swell task MoveDaRestart $config -d $datetime -m {{model_component}}" [[SaveRestart-{{model_component}}]] - script = "swell_task SaveRestart $config -d $datetime -m {{model_component}}" + script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" [[StageJedi-{{model_component}}]] - script = "swell_task StageJedi $config -m {{model_component}}" + script = "swell task StageJedi $config -m {{model_component}}" [[StageJediCycle-{{model_component}}]] - script = "swell_task StageJedi $config -d $datetime -m {{model_component}}" + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" [[GetObservations-{{model_component}}]] - script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" [[GenerateBClimatologyByLinking-{{model_component}}]] - script = "swell_task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" + script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell_task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -236,10 +236,10 @@ {% endif %} [[EvaJediLog-{{model_component}}]] - script = "swell_task EvaJediLog $config -d $datetime -m {{model_component}}" + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" [[EvaObservations-{{model_component}}]] - script = "swell_task EvaObservations $config -d $datetime -m {{model_component}}" + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} [[[directives]]] @@ -254,13 +254,13 @@ {% endif %} [[SaveObsDiags-{{model_component}}]] - script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" [[PrepareAnalysis-{{model_component}}]] - script = "swell_task PrepareAnalysis $config -d $datetime -m {{model_component}}" + script = "swell task PrepareAnalysis $config -d $datetime -m {{model_component}}" [[CleanCycle-{{model_component}}]] - script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/flow.cylc b/src/swell/suites/build_geos/flow.cylc index 8923ed33..16455d48 100644 --- a/src/swell/suites/build_geos/flow.cylc +++ b/src/swell/suites/build_geos/flow.cylc @@ -39,13 +39,13 @@ # Tasks # ----- [[CloneGeos]] - script = "swell_task CloneGeos $config" + script = "swell task CloneGeos $config" [[BuildGeosByLinking]] - script = "swell_task BuildGeosByLinking $config" + script = "swell task BuildGeosByLinking $config" [[BuildGeos]] - script = "swell_task BuildGeos $config" + script = "swell task BuildGeos $config" platform = {{platform}} execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} [[[directives]]] diff --git a/src/swell/suites/build_jedi/flow.cylc b/src/swell/suites/build_jedi/flow.cylc index 13760ee8..2f673b79 100644 --- a/src/swell/suites/build_jedi/flow.cylc +++ b/src/swell/suites/build_jedi/flow.cylc @@ -39,13 +39,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] diff --git a/src/swell/suites/convert_ncdiags/flow.cylc b/src/swell/suites/convert_ncdiags/flow.cylc index 0f6fbd58..c437e4df 100644 --- a/src/swell/suites/convert_ncdiags/flow.cylc +++ b/src/swell/suites/convert_ncdiags/flow.cylc @@ -70,13 +70,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -91,18 +91,18 @@ {% endif %} [[ GetGsiBc ]] - script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" [[ GsiBcToIoda ]] - script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" [[ GetGsiNcdiag ]] - script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" [[ GsiNcdiagToIoda ]] - script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" [[CleanCycle]] - script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" + script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/flow.cylc b/src/swell/suites/forecast_geos/flow.cylc index 4bffa71e..8b40bd20 100644 --- a/src/swell/suites/forecast_geos/flow.cylc +++ b/src/swell/suites/forecast_geos/flow.cylc @@ -75,13 +75,13 @@ # Tasks # ----- [[CloneGeos]] - script = "swell_task CloneGeos $config" + script = "swell task CloneGeos $config" [[BuildGeosByLinking]] - script = "swell_task BuildGeosByLinking $config" + script = "swell task BuildGeosByLinking $config" [[BuildGeos]] - script = "swell_task BuildGeos $config" + script = "swell task BuildGeos $config" platform = {{platform}} execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} [[[directives]]] @@ -95,22 +95,22 @@ {% endif %} [[PrepGeosRunDir]] - script = "swell_task PrepGeosRunDir $config -d $datetime" + script = "swell task PrepGeosRunDir $config -d $datetime" [[RemoveForecastDir]] - script = "swell_task RemoveForecastDir $config -d $datetime" + script = "swell task RemoveForecastDir $config -d $datetime" [[GetGeosRestart]] - script = "swell_task GetGeosRestart $config -d $datetime" + script = "swell task GetGeosRestart $config -d $datetime" [[MoveForecastRestart]] - script = "swell_task MoveForecastRestart $config -d $datetime" + script = "swell task MoveForecastRestart $config -d $datetime" [[SaveRestart]] - script = "swell_task SaveRestart $config -d $datetime" + script = "swell task SaveRestart $config -d $datetime" [[RunGeosExecutable]] - script = "swell_task RunGeosExecutable $config -d $datetime" + script = "swell task RunGeosExecutable $config -d $datetime" platform = {{platform}} execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} [[[directives]]] diff --git a/src/swell/suites/geosadas/flow.cylc b/src/swell/suites/geosadas/flow.cylc index 888d91ec..f92cba9c 100644 --- a/src/swell/suites/geosadas/flow.cylc +++ b/src/swell/suites/geosadas/flow.cylc @@ -70,31 +70,31 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[StageJedi]] - script = "swell_task StageJedi $config -m geos_atmosphere" + script = "swell task StageJedi $config -m geos_atmosphere" [[ GetGsiBc ]] - script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" [[ GsiBcToIoda ]] - script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" [[ GetGsiNcdiag ]] - script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" [[ GsiNcdiagToIoda ]] - script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" [[ GetGeosAdasBackground ]] - script = "swell_task GetGeosAdasBackground $config -d $datetime -m geos_atmosphere" + script = "swell task GetGeosAdasBackground $config -d $datetime -m geos_atmosphere" [[RunJediVariationalExecutable]] - script = "swell_task RunJediVariationalExecutable $config -d $datetime -m geos_atmosphere" + script = "swell task RunJediVariationalExecutable $config -d $datetime -m geos_atmosphere" platform = {{platform}} execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -109,6 +109,6 @@ {% endif %} [[CleanCycle]] - script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" + script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index f9494ac6..7a82a323 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -94,13 +94,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -116,19 +116,19 @@ {% for model_component in model_components %} [[StageJedi-{{model_component}}]] - script = "swell_task StageJedi $config -m {{model_component}}" + script = "swell task StageJedi $config -m {{model_component}}" [[StageJediCycle-{{model_component}}]] - script = "swell_task StageJedi $config -d $datetime -m {{model_component}}" + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" [[ GetBackground-{{model_component}} ]] - script = "swell_task GetBackground $config -d $datetime -m {{model_component}}" + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" [[GetObservations-{{model_component}}]] - script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" [[RunJediHofxExecutable-{{model_component}}]] - script = "swell_task RunJediHofxExecutable $config -d $datetime -m {{model_component}}" + script = "swell task RunJediHofxExecutable $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["RunJediHofxExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -143,7 +143,7 @@ {% endif %} [[EvaObservations-{{model_component}}]] - script = "swell_task EvaObservations $config -d $datetime -m {{model_component}}" + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} [[[directives]]] @@ -158,10 +158,10 @@ {% endif %} [[SaveObsDiags-{{model_component}}]] - script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" [[CleanCycle-{{model_component}}]] - script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/letkf/flow.cylc b/src/swell/suites/letkf/flow.cylc index 07ca8d10..2b787151 100644 --- a/src/swell/suites/letkf/flow.cylc +++ b/src/swell/suites/letkf/flow.cylc @@ -94,13 +94,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -116,19 +116,19 @@ {% for model_component in model_components %} [[StageJedi-{{model_component}}]] - script = "swell_task StageJedi $config -m {{model_component}}" + script = "swell task StageJedi $config -m {{model_component}}" [[StageJediCycle-{{model_component}}]] - script = "swell_task StageJedi $config -d $datetime -m {{model_component}}" + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" [[ GetEnsemble-{{model_component}} ]] - script = "swell_task GetEnsemble $config -d $datetime -m {{model_component}}" + script = "swell task GetEnsemble $config -d $datetime -m {{model_component}}" [[GetObservations-{{model_component}}]] - script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" [[RunJediLetkfExecutable-{{model_component}}]] - script = "swell_task RunJediLetkfExecutable $config -d $datetime -m {{model_component}}" + script = "swell task RunJediLetkfExecutable $config -d $datetime -m {{model_component}}" platform = {{platform}} execution time limit = {{scheduling["RunJediLetkfExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -143,13 +143,13 @@ {% endif %} [[EvaObservations-{{model_component}}]] - script = "swell_task EvaObservations $config -d $datetime -m {{model_component}}" + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" [[SaveObsDiags-{{model_component}}]] - script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" [[CleanCycle-{{model_component}}]] - script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index bdfcbff8..b6c51e42 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -81,13 +81,13 @@ # Tasks # ----- [[CloneJedi]] - script = "swell_task CloneJedi $config" + script = "swell task CloneJedi $config" [[BuildJediByLinking]] - script = "swell_task BuildJediByLinking $config" + script = "swell task BuildJediByLinking $config" [[BuildJedi]] - script = "swell_task BuildJedi $config" + script = "swell task BuildJedi $config" platform = {{platform}} execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} [[[directives]]] @@ -102,22 +102,22 @@ {% endif %} [[ GetGsiBc ]] - script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" [[ GsiBcToIoda ]] - script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" [[ GetGsiNcdiag ]] - script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" [[ GsiNcdiagToIoda ]] - script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" [[ GetGeovals ]] - script = "swell_task GetGeovals $config -d $datetime -m geos_atmosphere" + script = "swell task GetGeovals $config -d $datetime -m geos_atmosphere" [[RunJediUfoTestsExecutable]] - script = "swell_task RunJediUfoTestsExecutable $config -d $datetime -m geos_atmosphere" + script = "swell task RunJediUfoTestsExecutable $config -d $datetime -m geos_atmosphere" platform = {{platform}} execution time limit = {{scheduling["RunJediUfoTestsExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -132,7 +132,7 @@ {% endif %} [[EvaObservations]] - script = "swell_task EvaObservations $config -d $datetime -m geos_atmosphere" + script = "swell task EvaObservations $config -d $datetime -m geos_atmosphere" platform = {{platform}} execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} [[[directives]]] @@ -147,6 +147,6 @@ {% endif %} [[CleanCycle]] - script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" + script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py new file mode 100644 index 00000000..fd720f9a --- /dev/null +++ b/src/swell/swell.py @@ -0,0 +1,226 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import click + +from swell.deployment.platforms.platforms import get_platforms +from swell.deployment.create_experiment import clone_config, create_experiment_directory +from swell.deployment.create_experiment import prepare_config +from swell.deployment.launch_experiment import launch_experiment +from swell.tasks.base.task_base import task_wrapper, get_tasks +from swell.test.test_driver import test_wrapper, valid_tests +from swell.utilities.suite_utils import get_suites +from swell.utilities.welcome_message import write_welcome_message +from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper + + +# -------------------------------------------------------------------------------------------------- + + +@click.group() +def swell_driver(): + """ + Welcome to swell! + + This is the top level driver for swell. It serves as a container for various commands + related to experiment creation, launching, tasks, and utilities. + + The normal process for createing and running an experiment is to issue: + + swell create + + followed by + + swell launch + + """ + pass + + +# -------------------------------------------------------------------------------------------------- + +# Help strings for optional arguments + +input_method_help = 'Method by which to create the YAML configuration file. If choosing ' + \ + 'defaults the setting for the default suite test will be used. If using ' + \ + 'CLI you will be led through the questions to configure the experiment.' + +platform_help = 'If using defaults for input_method, this option is used to determine which ' + \ + 'platform to use for platform specific defaults. Options are ' + \ + str(get_platforms()) + +override_help = 'After generating the config file, parameters inside can be overridden ' + \ + 'using values from the override config file.' + +advanced_help = 'Show configuration questions which are otherwise not shown to the user.' + +no_detach_help = 'Tells the workflow manager not to detach. That is to say run the entire ' + \ + 'run the entire workflow in the foreground and pass back a return code.' + +log_path_help = 'Directory to receive workflow manager logging output (instead of ' + \ + '$HOME/cylc-run/)' + +datetime_help = 'Datetime to use for task execution. Format is yyyy-mm-ddThh:mm:ss. Note that ' + \ + 'non-numeric characters will be stripped from the string. Minutes and seconds ' + \ + 'are optional.' + +model_help = 'Data assimilation system. I.e. the model being initialized by data assimilation.' + + +# -------------------------------------------------------------------------------------------------- + + +@swell_driver.command() +@click.argument('suite', type=click.Choice(get_suites())) +@click.option('-m', '--input_method', 'input_method', default='defaults', + type=click.Choice(['defaults', 'cli']), help=input_method_help) +@click.option('-p', '--platform', 'platform', default='nccs_discover', + type=click.Choice(get_platforms()), help=platform_help) +@click.option('-o', '--override', 'override', default=None, help=override_help) +@click.option('-a', '--advanced', 'advanced', default=False, help=advanced_help) +def create(suite, input_method, platform, override, advanced): + """ + Create a new experiment + + This command creates an experiment directory based on the provided suite name and options. + + Arguments: \n + suite (str): Name of the suite you wish to run. \n + + """ + # First create the configuration for the experiment. + experiment_dict_str = prepare_config(suite, input_method, platform, override, advanced) + + # Create the experiment directory + create_experiment_directory(experiment_dict_str) + + +# -------------------------------------------------------------------------------------------------- + + +@swell_driver.command() +@click.argument('configuration') +@click.argument('experiment_id') +@click.option('-m', '--input_method', 'input_method', default='defaults', + type=click.Choice(['defaults', 'cli']), help=input_method_help) +@click.option('-p', '--platform', 'platform', default=None, help=platform_help) +@click.option('-a', '--advanced', 'advanced', default=False, help=advanced_help) +def clone(configuration, experiment_id, input_method, platform, advanced): + """ + Clone an existing experiment + + This command creates an experiment directory based on the provided experiment configuration. + + Arguments: \n + configuration (str): Path to a YAML containing the experiment configuration you wish to + clone from. \n + + """ + # Create experiment configuration by cloning from existing experiment + experiment_dict_str = clone_config(configuration, experiment_id, input_method, platform, + advanced) + + # Create the experiment directory + create_experiment_directory(experiment_dict_str) + + +# -------------------------------------------------------------------------------------------------- + + +@swell_driver.command() +@click.argument('suite_path') +@click.option('-b', '--no-detach', 'no_detach', is_flag=True, default=False, help=no_detach_help) +@click.option('-l', '--log_path', 'log_path', default=None, help=log_path_help) +def launch(suite_path, no_detach, log_path): + """ + Launch an experiment with the cylc workflow manager + + This command launches an experiment using the provided suite path and options. + + Arguments: \n + suite_path (str): Path to where the flow.cylc and associated suite files are located. \n + + """ + launch_experiment(suite_path, no_detach, log_path) + + +# -------------------------------------------------------------------------------------------------- + + +@swell_driver.command() +@click.argument('task', type=click.Choice(get_tasks())) +@click.argument('config') +@click.option('-d', '--datetime', 'datetime', default=None, help=datetime_help) +@click.option('-m', '--model', 'model', default=None, help=model_help) +def task(task, config, datetime, model): + """ + Run a workflow task + + This command executes a task using the provided task name, configuration file and options. + + Arguments:\n + task (str): Name of the task to execute.\n + config (str): Path to the configuration file for the task.\n + + """ + task_wrapper(task, config, datetime, model) + + +# -------------------------------------------------------------------------------------------------- + + +@swell_driver.command() +@click.argument('utility', type=click.Choice(get_utilities())) +def utility(utility): + """ + Run a utility script + + This command performs a utility operation specified by the utility argument. + + Arguments:\n + utility (str): Name of the utility operation to perform.\n + + """ + utility_wrapper(utility) + + +# -------------------------------------------------------------------------------------------------- + + +@swell_driver.command() +@click.argument('test', type=click.Choice(valid_tests)) +def test(test): + """ + Run one of the test suites + + This command performs the test specified by the test argument. + + Arguments:\n + test (str): Name of the test to execute. + + """ + test_wrapper(test) + + +# -------------------------------------------------------------------------------------------------- + + +def main(): + """ + Main Function + + This function is the entry point for swell. It writes a welcome message and + sets up the driver group. + """ + write_welcome_message() + swell_driver() + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index e1d1c039..27e54ba8 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -11,14 +11,14 @@ # standard imports from abc import ABC, abstractmethod -import click +import glob import importlib import os import time # swell imports -from swell.tasks.base.task_registry import valid_tasks -from swell.utilities.case_switching import camel_case_to_snake_case +from swell.swell_path import get_swell_path +from swell.utilities.case_switching import camel_case_to_snake_case, snake_case_to_camel_case from swell.utilities.config import Config from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams from swell.utilities.datetime import Datetime @@ -224,19 +224,28 @@ def create_task(self, task, config, datetime, model): # -------------------------------------------------------------------------------------------------- +def get_tasks(): -def task_main(task, config, datetime, model): + # Path to tasks + tasks_directory = os.path.join(get_swell_path(), 'tasks', '*.py') - # For security check that task is in the registry - if task not in valid_tasks: - valid_task_logger = Logger('CheckValidTasks') - valid_task_logger.info(' ') - valid_task_logger.info('Task \'' + task + '\' not found in registry; valid tasks are:') - valid_tasks.sort() - for valid_task in valid_tasks: - valid_task_logger.info(' ' + valid_task) - valid_task_logger.info(' ') - valid_task_logger.abort('ABORT: Task not found in task registry.') + # List of tasks + task_files = sorted(glob.glob(tasks_directory)) + + # Get just the task name + tasks = [] + for task_file in task_files: + base_name = os.path.basename(task_file) + if '__' not in base_name: + tasks.append(snake_case_to_camel_case(base_name[0:-3])) + + # Return list of valid task choices + return tasks + +# -------------------------------------------------------------------------------------------------- + + +def task_wrapper(task, config, datetime, model): # Create the object constrc_start = time.perf_counter() @@ -262,23 +271,3 @@ def task_main(task, config, datetime, model): # -------------------------------------------------------------------------------------------------- - - -@click.command() -@click.argument('task') -@click.argument('config') -@click.option('-d', '--datetime', 'datetime', default=None) -@click.option('-m', '--model', 'model', default=None) -def main(task, config, datetime, model): - - task_main(task, config, datetime, model) - - -# -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py deleted file mode 100644 index 4683e87a..00000000 --- a/src/swell/tasks/base/task_registry.py +++ /dev/null @@ -1,46 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -valid_tasks = [ - 'BuildGeosByLinking', - 'BuildGeos', - 'BuildJediByLinking', - 'BuildJedi', - 'CleanCycle', - 'CloneGeos', - 'CloneJedi', - 'EvaJediLog', - 'EvaObservations', - 'GenerateBClimatologyByLinking', - 'GenerateBClimatology', - 'GetBackgroundGeosExperiment', - 'GetBackground', - 'GetEnsemble', - 'GetGeosAdasBackground', - 'GetGeosRestart', - 'GetGeovals', - 'GetGsiBc', - 'GetGsiNcdiag', - 'GetObservations', - 'GsiBcToIoda', - 'GsiNcdiagToIoda', - 'LinkGeosOutput', - 'MoveDaRestart', - 'MoveForecastRestart', - 'ObsProcessSetup', - 'PrepareAnalysis', - 'PrepGeosRunDir', - 'RemoveForecastDir', - 'RunGeosExecutable', - 'RunJediHofxExecutable', - 'RunJediLetkfExecutable', - 'RunJediUfoTestsExecutable', - 'RunJediVariationalExecutable', - 'SaveObsDiags', - 'SaveRestart', - 'StageJedi', - 'StoreBackground', -] diff --git a/src/swell/test/code_tests/code_tests.py b/src/swell/test/code_tests/code_tests.py index 062dc05a..47bf5311 100644 --- a/src/swell/test/code_tests/code_tests.py +++ b/src/swell/test/code_tests/code_tests.py @@ -18,7 +18,7 @@ # -------------------------------------------------------------------------------------------------- -def main(): +def code_tests(): # Create a logger logger = Logger('TestSuite') diff --git a/src/swell/test/code_tests/question_dictionary_comparison_test.py b/src/swell/test/code_tests/question_dictionary_comparison_test.py index 066c99a6..fcec99b9 100644 --- a/src/swell/test/code_tests/question_dictionary_comparison_test.py +++ b/src/swell/test/code_tests/question_dictionary_comparison_test.py @@ -10,8 +10,8 @@ import unittest -from swell.utilities.bin.task_question_dicts import tq_dicts -from swell.utilities.bin.task_question_dicts_defaults import tq_dicts_defaults +from swell.utilities.scripts.task_question_dicts import main as tq_dicts +from swell.utilities.scripts.task_question_dicts_defaults import main as tq_dicts_defaults # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/test/test_driver.py b/src/swell/test/test_driver.py new file mode 100644 index 00000000..414e89b5 --- /dev/null +++ b/src/swell/test/test_driver.py @@ -0,0 +1,30 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +import importlib + +# -------------------------------------------------------------------------------------------------- + +valid_tests = ['code_tests'] + +# -------------------------------------------------------------------------------------------------- + + +def test_wrapper(test): + + # Test script + test_script_file = 'swell.test.'+test+'.'+test + + # Import the correct method + test_method = getattr(importlib.import_module(test_script_file), test) + + # Run the test + test_method() + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/__init__.py b/src/swell/utilities/bin/__init__.py deleted file mode 100644 index f82722ea..00000000 --- a/src/swell/utilities/bin/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -import os - -repo_directory = os.path.dirname(__file__) diff --git a/src/swell/deployment/bin/__init__.py b/src/swell/utilities/scripts/__init__.py similarity index 100% rename from src/swell/deployment/bin/__init__.py rename to src/swell/utilities/scripts/__init__.py diff --git a/src/swell/utilities/bin/check_jedi_interface_templates.py b/src/swell/utilities/scripts/check_jedi_interface_templates.py similarity index 95% rename from src/swell/utilities/bin/check_jedi_interface_templates.py rename to src/swell/utilities/scripts/check_jedi_interface_templates.py index 5579e62a..aad74256 100644 --- a/src/swell/utilities/bin/check_jedi_interface_templates.py +++ b/src/swell/utilities/scripts/check_jedi_interface_templates.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -98,10 +96,3 @@ def main(): # -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - main() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/bin/swell_sat_db_processing.py b/src/swell/utilities/scripts/swell_sat_db_processing.py similarity index 98% rename from src/swell/deployment/bin/swell_sat_db_processing.py rename to src/swell/utilities/scripts/swell_sat_db_processing.py index d1e17c18..62cafa8c 100644 --- a/src/swell/deployment/bin/swell_sat_db_processing.py +++ b/src/swell/utilities/scripts/swell_sat_db_processing.py @@ -90,7 +90,3 @@ def main(config): # create yamls make_yamls(processed_data, yaml_out_dir) - - -if __name__ == '__main__': - main() diff --git a/src/swell/utilities/bin/task_question_dicts.py b/src/swell/utilities/scripts/task_question_dicts.py similarity index 97% rename from src/swell/utilities/bin/task_question_dicts.py rename to src/swell/utilities/scripts/task_question_dicts.py index b2e94c8c..c991cf1d 100644 --- a/src/swell/utilities/bin/task_question_dicts.py +++ b/src/swell/utilities/scripts/task_question_dicts.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -25,7 +23,7 @@ # -------------------------------------------------------------------------------------------------- -def tq_dicts(): +def main(): # Create a logger logger = Logger('ListOfTaskQuestions') @@ -162,10 +160,3 @@ def tq_dicts(): # -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - tq_dicts() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/task_question_dicts_defaults.py b/src/swell/utilities/scripts/task_question_dicts_defaults.py similarity index 98% rename from src/swell/utilities/bin/task_question_dicts_defaults.py rename to src/swell/utilities/scripts/task_question_dicts_defaults.py index 8c065263..24349d72 100644 --- a/src/swell/utilities/bin/task_question_dicts_defaults.py +++ b/src/swell/utilities/scripts/task_question_dicts_defaults.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -126,7 +124,7 @@ def create_platform_tq_dicts(logger, platform_name, tq_dicts, platform_tq_dicts_ # -------------------------------------------------------------------------------------------------- -def tq_dicts_defaults(): +def main(): # Create a logger logger = Logger('ListOfTaskQuestions') @@ -248,10 +246,3 @@ def tq_dicts_defaults(): # -------------------------------------------------------------------------------------------------- - - -if __name__ == '__main__': - tq_dicts_defaults() - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/scripts/utility_driver.py b/src/swell/utilities/scripts/utility_driver.py new file mode 100644 index 00000000..7f58961c --- /dev/null +++ b/src/swell/utilities/scripts/utility_driver.py @@ -0,0 +1,62 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os +import importlib + +from swell.swell_path import get_swell_path +from swell.utilities.case_switching import snake_case_to_camel_case, camel_case_to_snake_case + + +# -------------------------------------------------------------------------------------------------- + + +def get_utilities(): + + # Path to util scripts + util_scripts_dir = os.path.join(get_swell_path(), 'utilities', 'scripts', '*.py') + + # List of tasks + util_script_files = sorted(glob.glob(util_scripts_dir)) + + # Get just the task name + util_scripts = [] + for util_script_file in util_script_files: + base_name = os.path.basename(util_script_file) + if '__' not in base_name: + util_scripts.append(snake_case_to_camel_case(base_name[0:-3])) + + # Remove UtilityDriver from util_scripts + util_scripts.remove('UtilityDriver') + + # Return list of valid task choices + return util_scripts + + +# -------------------------------------------------------------------------------------------------- + + +def utility_wrapper(utility): + + # Convert utility to snake case + utility_snake = camel_case_to_snake_case(utility) + + # Test script + test_script_file = 'swell.utilities.scripts.'+utility_snake + + # Import the correct method + utility_method = getattr(importlib.import_module(test_script_file), 'main') + + # Run the test + utility_method() + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py new file mode 100644 index 00000000..9abc856a --- /dev/null +++ b/src/swell/utilities/suite_utils.py @@ -0,0 +1,59 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os + +from swell.swell_path import get_swell_path + + +# -------------------------------------------------------------------------------------------------- + + +def get_suites(): + + # Path to platforms + suites_directory = os.path.join(get_swell_path(), 'suites') + + # List of suites + suites = [dir for dir in os.listdir(suites_directory) + if os.path.isdir(os.path.join(suites_directory, dir))] + + # Sort list alphabetically + suites = sorted(suites) + + # List all directories in platform_directory + return suites + + +# -------------------------------------------------------------------------------------------------- + + +def get_suite_tests(): + + # Path to platforms + suite_tests_directory = os.path.join(get_swell_path(), 'test', 'suite_tests', '*.yaml') + + # List of tasks + suite_test_files = sorted(glob.glob(suite_tests_directory)) + + # Get just the task name + suite_tests = [] + for suite_test_file in suite_test_files: + suite_tests.append(os.path.basename(suite_test_file)[0:-5]) + + # Sort list alphabetically + suite_tests = sorted(suite_tests) + + # Return list of valid task choices + return suite_tests + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/welcome_message.py b/src/swell/utilities/welcome_message.py index 2dfc5593..3b0c1a93 100644 --- a/src/swell/utilities/welcome_message.py +++ b/src/swell/utilities/welcome_message.py @@ -14,7 +14,7 @@ # -------------------------------------------------------------------------------------------------- -def write_welcome_message(application): +def write_welcome_message(): logger = Logger('') @@ -24,5 +24,4 @@ def write_welcome_message(application): logger.blank(f"\__ \\\ V V / __/ | | Version {__version__}", False) # noqa logger.blank(f"|___/ \_/\_/ \___|_|_| \x1B[4m\x1B[34mhttps://geos-esm.github.io/swell\033[0m", False) # noqa logger.blank(f" ", False) # noqa - logger.blank(f"Executing swell \'{application.lower()}\' application", False) # noqa logger.blank('', False) # noqa