Add option to save geovals

src/swell/tasks/run_jedi_hofx_executable.py

@@ -8,10 +8,13 @@
 # --------------------------------------------------------------------------------------------------
 
 
+import glob
 import os
 import yaml
+import xarray as xr
 
 from swell.tasks.base.task_base import taskBase
+from swell.utilities.netcdf_files import combine_files_without_groups
 from swell.utilities.run_jedi_executables import jedi_dictionary_iterator, run_executable
 
 
@@ -37,6 +40,7 @@ def execute(self):
         observations = self.config.observations()
         jedi_forecast_model = self.config.jedi_forecast_model(None)
         generate_yaml_and_exit = self.config.generate_yaml_and_exit(False)
+        save_geovals = self.config.save_geovals()
 
         # Set the observing system records path
         self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None))
@@ -95,6 +99,43 @@ def execute(self):
         jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations,
                                  jedi_forecast_model)
 
+        # If window type is 4D add time interpolation to each observer
+        # ------------------------------------------------------------
+        if window_type == '4D':
+            for observer in jedi_config_dict['observations']['observers']:
+                observer['get values'] = {
+                    'time interpolation': 'linear'
+                }
+
+        # Update config filters to save the GeoVaLs from the model interface
+        # ------------------------------------------------------------------
+        if save_geovals:
+
+            for index, observation in enumerate(observations):
+
+                # Define the GeoVaLs saver dictionary
+                gom_saver_dict = {
+                    'filter': 'GOMsaver',
+                    'filename': os.path.join(self.cycle_dir(),
+                                             f'{observation}-geovals.{window_begin}.nc4')
+                }
+
+                # Get pointer to observer
+                observer = jedi_config_dict['observations']['observers'][index]
+
+                # Check if observer has obs filters and if so add them to the jedi_config_dict
+                if 'obs filters' in observer:
+                    filter_dict = 'obs filters'
+                elif 'obs post filters' in observer:
+                    filter_dict = 'obs post filters'
+                else:
+                    # Create some post filters
+                    observer['obs post filters'] = []
+                    filter_dict = 'obs post filters'
+
+                # Append the GOMsaver dictionary to the observer filters
+                observer[filter_dict].append(gom_saver_dict)
+
         # Write the expanded dictionary to YAML file
         # ------------------------------------------
         with open(jedi_config_file, 'w') as jedi_config_file_open:
@@ -123,4 +164,31 @@ def execute(self):
         else:
             self.logger.info('YAML generated, now exiting.')
 
+        # If saving the geovals they need to be combined
+        # ----------------------------------------------
+        if save_geovals:
+
+            # Combine the GeoVaLs
+            # -------------------
+            for observation in observations:
+
+                self.logger.info('Combining GeoVaLs files for {observation}')
+
+                # List of GeoVaLs input files
+                input_files = f'{observation}-geovals.{window_begin}_*.nc4'
+                output_file = f'{self.experiment_id()}.{observation}-geovals.{window_begin}.nc4'
+                output_file = os.path.join(self.cycle_dir(), output_file)
+
+                # Build list of input files
+                geovals_files = sorted(glob.glob(os.path.join(self.cycle_dir(), input_files)))
+
+                # Assert that there are np files
+                self.logger.assert_abort(len(geovals_files) == np, f'Number of GeoVaLs files ' +
+                                         f'does not match number of processors.')
+
+                # Write the concatenated dataset to a new file
+
+                combine_files_without_groups(self.logger, geovals_files, output_file, 'nlocs', True)
+
+
 # --------------------------------------------------------------------------------------------------

src/swell/tasks/task_questions.yaml

@@ -684,6 +684,17 @@ produce_geovals:
   - GsiNcdiagToIoda
   type: boolean
 
+save_geovals:
+  ask_question: false
+  default_value: false
+  options:
+  - true
+  - false
+  prompt: When running hofx do you want to output the GeoVaLs?
+  tasks:
+  - RunJediHofxExecutable
+  type: boolean
+
 single_observations:
   ask_question: false
   default_value: false

src/swell/test/suite_tests/hofx-tier1.yaml

@@ -1,6 +1,7 @@
 jedi_build_method: use_existing
 existing_jedi_source_directory: /discover/nobackup/gmao_ci/swell/tier2/stable/build_jedi/jedi_bundle/source/
 existing_jedi_build_directory: /discover/nobackup/gmao_ci/swell/tier2/stable/build_jedi/jedi_bundle/build/
+save_geovals: true
 models:
   geos_atmosphere:
     horizontal_resolution: '91'
@@ -40,3 +41,6 @@ models:
     - ssmis_f17
     obs_experiment: x0048v2
     geovals_experiment: x0048v2-geovals
+    clean_patterns: []
+
+

src/swell/test/suite_tests/ufo_testing-tier1.yaml

@@ -44,7 +44,6 @@ models:
     - ssmis_f17
     produce_geovals: false
     clean_patterns:
-    - '*.nc4'
     - '*.txt'
     - '*.log'
     - '*.yaml'

src/swell/utilities/netcdf_files.py

@@ -0,0 +1,42 @@
+# (C) Copyright 2021- United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+import os
+import xarray as xr
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+def combine_files_without_groups(logger, list_of_input_files, output_file, concat_dim,
+                                 delete_input=False):
+
+    # Write some information
+    logger.info('Combining the following netCDF files (using no-group combine): ')
+    for f in list_of_input_files:
+        logger.info(f' - {f}', False)
+    logger.info(f'Writing to file {output_file}')
+
+    # Load the files as Xarray datasets
+    datasets = [xr.open_dataset(f) for f in list_of_input_files]
+
+    # Concatenate the datasets along the 'nlocs' dimension
+    concatenated_ds = xr.concat(datasets, dim=concat_dim)
+
+    # Write the concatenated dataset to a new file
+    concatenated_ds.to_netcdf(output_file)
+
+    # Delete the input files if requested
+    if delete_input:
+        for f in list_of_input_files:
+            os.remove(f)
+
+
+# --------------------------------------------------------------------------------------------------