From 7cdaa4c0618a2e487c400012ebced7296b918635 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 13 Dec 2023 13:39:53 -0500 Subject: [PATCH 1/2] Add option to save geovals --- src/swell/tasks/run_jedi_hofx_executable.py | 68 +++++++++++++++++++ src/swell/tasks/task_questions.yaml | 11 +++ src/swell/test/suite_tests/hofx-tier1.yaml | 4 ++ .../test/suite_tests/ufo_testing-tier1.yaml | 1 - src/swell/utilities/netcdf_files.py | 42 ++++++++++++ 5 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 src/swell/utilities/netcdf_files.py diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 75d75530..2628750d 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -8,10 +8,13 @@ # -------------------------------------------------------------------------------------------------- +import glob import os import yaml +import xarray as xr from swell.tasks.base.task_base import taskBase +from swell.utilities.netcdf_files import combine_files_without_groups from swell.utilities.run_jedi_executables import jedi_dictionary_iterator, run_executable @@ -37,6 +40,7 @@ def execute(self): observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + save_geovals = self.config.save_geovals() # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) @@ -95,6 +99,43 @@ def execute(self): jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, jedi_forecast_model) + # If window type is 4D add time interpolation to each observer + # ------------------------------------------------------------ + if window_type == '4D': + for observer in jedi_config_dict['observations']['observers']: + observer['get values'] = { + 'time interpolation': 'linear' + } + + # Update config filters to save the GeoVaLs from the model interface + # ------------------------------------------------------------------ + if save_geovals: + + for index, observation in enumerate(observations): + + # Define the GeoVaLs saver dictionary + gom_saver_dict = { + 'filter': 'GOMsaver', + 'filename': os.path.join(self.cycle_dir(), + f'{observation}-geovals.{window_begin}.nc4') + } + + # Get pointer to observer + observer = jedi_config_dict['observations']['observers'][index] + + # Check if observer has obs filters and if so add them to the jedi_config_dict + if 'obs filters' in observer: + filter_dict = 'obs filters' + elif 'obs post filters' in observer: + filter_dict = 'obs post filters' + else: + # Create some post filters + observer['obs post filters'] = [] + filter_dict = 'obs post filters' + + # Append the GOMsaver dictionary to the observer filters + observer[filter_dict].append(gom_saver_dict) + # Write the expanded dictionary to YAML file # ------------------------------------------ with open(jedi_config_file, 'w') as jedi_config_file_open: @@ -123,4 +164,31 @@ def execute(self): else: self.logger.info('YAML generated, now exiting.') + # If saving the geovals they need to be combined + # ---------------------------------------------- + if save_geovals: + + # Combine the GeoVaLs + # ------------------- + for observation in observations: + + self.logger.info('Combining GeoVaLs files for {observation}') + + # List of GeoVaLs input files + input_files = f'{observation}-geovals.{window_begin}_*.nc4' + output_file = f'{self.experiment_id()}.{observation}-geovals.{window_begin}.nc4' + output_file = os.path.join(self.cycle_dir(), output_file) + + # Build list of input files + geovals_files = sorted(glob.glob(os.path.join(self.cycle_dir(), input_files))) + + # Assert that there are np files + self.logger.assert_abort(len(geovals_files) == np, f'Number of GeoVaLs files ' + + f'does not match number of processors.') + + # Write the concatenated dataset to a new file + + combine_files_without_groups(self.logger, geovals_files, output_file, 'nlocs', True) + + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 5dcdcd10..0541dfc7 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -684,6 +684,17 @@ produce_geovals: - GsiNcdiagToIoda type: boolean +save_geovals: + ask_question: false + default_value: false + options: + - true + - false + prompt: When running hofx do you want to output the GeoVaLs? + tasks: + - RunJediHofxExecutable + type: boolean + single_observations: ask_question: false default_value: false diff --git a/src/swell/test/suite_tests/hofx-tier1.yaml b/src/swell/test/suite_tests/hofx-tier1.yaml index 0cf073a8..0eae366b 100644 --- a/src/swell/test/suite_tests/hofx-tier1.yaml +++ b/src/swell/test/suite_tests/hofx-tier1.yaml @@ -1,6 +1,7 @@ jedi_build_method: use_existing existing_jedi_source_directory: /discover/nobackup/gmao_ci/swell/tier2/stable/build_jedi/jedi_bundle/source/ existing_jedi_build_directory: /discover/nobackup/gmao_ci/swell/tier2/stable/build_jedi/jedi_bundle/build/ +save_geovals: true models: geos_atmosphere: horizontal_resolution: '91' @@ -40,3 +41,6 @@ models: - ssmis_f17 obs_experiment: x0048v2 geovals_experiment: x0048v2-geovals + clean_patterns: [] + + diff --git a/src/swell/test/suite_tests/ufo_testing-tier1.yaml b/src/swell/test/suite_tests/ufo_testing-tier1.yaml index 77c41361..82fa454e 100644 --- a/src/swell/test/suite_tests/ufo_testing-tier1.yaml +++ b/src/swell/test/suite_tests/ufo_testing-tier1.yaml @@ -44,7 +44,6 @@ models: - ssmis_f17 produce_geovals: false clean_patterns: - - '*.nc4' - '*.txt' - '*.log' - '*.yaml' diff --git a/src/swell/utilities/netcdf_files.py b/src/swell/utilities/netcdf_files.py new file mode 100644 index 00000000..8fef1a23 --- /dev/null +++ b/src/swell/utilities/netcdf_files.py @@ -0,0 +1,42 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import xarray as xr + + +# -------------------------------------------------------------------------------------------------- + + +def combine_files_without_groups(logger, list_of_input_files, output_file, concat_dim, + delete_input=False): + + # Write some information + logger.info('Combining the following netCDF files (using no-group combine): ') + for f in list_of_input_files: + logger.info(f' - {f}', False) + logger.info(f'Writing to file {output_file}') + + # Load the files as Xarray datasets + datasets = [xr.open_dataset(f) for f in list_of_input_files] + + # Concatenate the datasets along the 'nlocs' dimension + concatenated_ds = xr.concat(datasets, dim=concat_dim) + + # Write the concatenated dataset to a new file + concatenated_ds.to_netcdf(output_file) + + # Delete the input files if requested + if delete_input: + for f in list_of_input_files: + os.remove(f) + + +# -------------------------------------------------------------------------------------------------- From e3ba302089ff5a6734e00ea08c9af5f691e5a495 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 21 Dec 2023 10:52:04 -0500 Subject: [PATCH 2/2] remove an unused variable --- src/swell/tasks/run_jedi_hofx_executable.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 2628750d..e0b24279 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -11,7 +11,6 @@ import glob import os import yaml -import xarray as xr from swell.tasks.base.task_base import taskBase from swell.utilities.netcdf_files import combine_files_without_groups